Experimental speedy depchecker patch

Here's a first stab at a patch that replaces depchecker's network bound
slow GNOME FTP scraping with parsing of "versions" formatted files.
Earlier we could have gotten the release versions quickly this way as
well, cutting the time spent on scraping to half, but now vuntz made
server-size hourly versions file of latest versions for each gnome
version and we can parse that.
The URL is temporary and will change later.

This patch is for feedback purposes. To my dirty code and especially for
feedback to pass on to vuntz about the files that are in that /tmp
directory of his currently.

I've linked up my runs at http://dev.gentoo.org/~leio/gnome/ for
comparison of the FTP scraping and versions file results. the sorting
seems to be different a bit, but I don't have time to get that to match
right now. Heavy improvements possible, etc. I just did a first working
versions to see if the upstream file works out and will go back to GNOME
2.22 stuff. So, anyone else can improve on top of that - I won't revisit
it for some days.

For example there could be --stable and --unstable switches (latest
upstream stable release cycle, latest upstream development release
cycle). See
http://www.gnome.org/~vuntz/tmp/versions/

I don't think this should be pushed before the upstream URL settles and
we compare if the results are not worse, etc.
I think I also have no proper --nextrev handling in regards to the file
we need to use then.

commit ad04a2ee64ffbd8237c2dbd6847751f0a0caa482
Author: Mart Raudsepp <leio@gentoo.org>
Date:   Wed Mar 19 00:41:46 2008 +0200

    Use versions files generated in upstream server in an hourly basis instead of slow FTP scraping

Signed-off-by: Daniel Gryniewicz <dang@gentoo.org>

2 files changed, 55 insertions, 9 deletions

diff --git a/main.py b/main.py
index e76704b..bde51f9 100755
--- a/main.py
+++ b/main.py
@@ -18,9 +18,16 @@ if __name__ == '__main__':
     # 2nd list is the latest versions of the packages w.r.t. the major/minor of
     # the release ( from GNOME FTP )
     import gnome_module
+
     gnome = gnome_module.GNOME(options.get_arguments().nextrev)
-    release_packages, latest_packages = gnome.generate_data()
-    
+
+    # Quick versions file parsing
+    release_packages = gnome.generate_data_release()
+    latest_packages = gnome.generate_data_individual()
+
+    # Old FTP scraping way:
+    # release_packages, latest_packages = gnome.generate_data_ftp()
+
     # figure out what versions of these packages are in portage.
     # we need a list of package names to check for, so we choose
     # to use the release_packages list.
diff --git a/modules/gnome_module.py b/modules/gnome_module.py
index 78ef952..91d7c6b 100644
--- a/modules/gnome_module.py
+++ b/modules/gnome_module.py
@@ -2,8 +2,11 @@
 # LICENSE - GPL2
 
 # gnome module
-import ftp_module, package_module, string, simple_cache_module
+import urllib2, package_module, string, simple_cache_module
 import clioptions_module
+# import ftp_module
+
+
 
 DEBUG=False
 class GNOME:
@@ -14,15 +17,46 @@ class GNOME:
         self.nextrev = nextrev;
         self.major_release = ".".join( args.release_number.split(".")[:2] )
         self.full_release = args.release_number
+
+        # gnome_release is the major release number the development is leading up to or is part of. E.g, 2.21.5 will turn to 2.22 and 2.22.2 becomes 2.22
+        # This is used in latest versions list fetching URL construction
+        gnome_release_list = args.release_number.split(".")
+        if int(gnome_release_list[1]) % 2 != 0:
+            gnome_release_list[1] = str(int(gnome_release_list[1]) + 1)
+        self.gnome_release = ".".join(gnome_release_list[:2])
+
         self.ftpserver = "ftp.gnome.org"
         self.release_directories = ["pub/GNOME/admin/" + self.major_release + "/" + self.full_release + "/",
                                "pub/GNOME/platform/" + self.major_release + "/" + self.full_release + "/",
                                "pub/GNOME/desktop/" + self.major_release + "/" + self.full_release + "/",
                                "pub/GNOME/bindings/" + self.major_release + "/" + self.full_release + "/"]
-        
-    
-    def generate_data(self):
-        # connect to ftp and get the list of all the packages in the release directories    
+        self.latest_versions_file_path = 'http://www.gnome.org/~vuntz/tmp/versions/'
+        self.release_versions_file_path = 'http://ftp.gnome.org/pub/GNOME/teams/releng/'
+
+    def generate_data_from_versions_markup(self, url):
+        f = urllib2.urlopen(url)
+        ret = []
+        for line in f.readlines():
+            components = line.split(':')
+            if len(components) == 4 and len(components[2]) > 0:
+                # Upper modules category handling determination currently sucks - manually ignore modules that aren't unique $name
+                if components[1] not in ('gdl'):
+                    # Very hacky temporary way to ignore perl packages that don't have versions known in version-stable due to different infrastructure for hosting
+                    # We have a workaround in compare_packages now, but we don't have a proper mapping to ruby-g* stuff yet, so ignore them for now
+                    if not(components[0] == 'bindings' and components[1][0] == 'G'):
+                        ret.append(package_module.Package(components[1] + "-" + components[2]))
+        f.close()
+        return ret
+
+    def generate_data_individual(self):
+        return self.generate_data_from_versions_markup(self.latest_versions_file_path + 'versions-' + self.gnome_release)
+
+    def generate_data_release(self):
+        return self.generate_data_from_versions_markup(self.release_versions_file_path + self.full_release + '/versions')
+
+    """ We might want to modify this later to an extra fallback and/or for non-official modules whose tarballs are hosted on GNOME or compatible FTP
+    def generate_data_ftp(self):
+        # Deprecated: connect to ftp and get the list of all the packages in the release directories
         walker = ftp_module.FTPWalker(self.ftpserver,"anonymous","test@test.com")
         
         files = []
@@ -91,7 +125,8 @@ class GNOME:
         cache.flush_queue()
         
         return (release_packages, latest_packages)
-    
+    """
+
     def filter_files(self, files):
         # we want to filter out all the bad files.
         newfiles = []
@@ -161,8 +196,12 @@ def compare_packages(release_packages, latest_packages, packages_in_portage):
         else:
             status = package_module.Status.Compliant
 
-        
         #if portage_package != None:
+
+        # FIXME: Reports release version as latest version to not have to deal with this right now
+        if latest_package == None:
+            print "No latest version known for %s, FIXME!" % release_package.name
+            latest_package = release_package
             
         if DEBUG:
             print "package: " + str(release_package.name) + \