diff options
author | 2015-08-04 22:54:21 +0200 | |
---|---|---|
committer | 2015-08-05 20:32:11 +0200 | |
commit | 29875e07c1749ddf895d2058046c17092d72732e (patch) | |
tree | 1b658004588265a09c979355ce1902c0128facf9 | |
parent | [docs] update documentation (diff) | |
download | g-sorcery-29875e07c1749ddf895d2058046c17092d72732e.tar.gz g-sorcery-29875e07c1749ddf895d2058046c17092d72732e.tar.bz2 g-sorcery-29875e07c1749ddf895d2058046c17092d72732e.zip |
[g_sorcery/package_db] new DB syncing
-rw-r--r-- | docs/developer_instructions.html | 3 | ||||
-rw-r--r-- | docs/developer_instructions.rst | 3 | ||||
-rw-r--r-- | docs/g-sorcery.8 | 2 | ||||
-rw-r--r-- | docs/g-sorcery.8.rst | 2 | ||||
-rw-r--r-- | docs/g-sorcery.cfg.8 | 2 | ||||
-rw-r--r-- | docs/g-sorcery.cfg.8.rst | 2 | ||||
-rw-r--r-- | g_sorcery/backend.py | 47 | ||||
-rw-r--r-- | g_sorcery/git_syncer/__init__.py | 1 | ||||
-rw-r--r-- | g_sorcery/git_syncer/git_syncer.py | 67 | ||||
-rw-r--r-- | g_sorcery/package_db.py | 60 | ||||
-rw-r--r-- | g_sorcery/syncer.py | 107 | ||||
-rw-r--r-- | setup.py | 4 | ||||
-rw-r--r-- | tests/test_PackageDB.py | 9 |
13 files changed, 247 insertions, 62 deletions
diff --git a/docs/developer_instructions.html b/docs/developer_instructions.html index 4cea319..4af5e35 100644 --- a/docs/developer_instructions.html +++ b/docs/developer_instructions.html @@ -581,7 +581,8 @@ backends). PackageDB class API should be used instead.</p> to add categories and packages and to do queries on them. Usually you do not want to customize this class.</p> <p>If you have a database that should be synced with another already generate database -you can redifine URI to be used for syncing using <strong>get_real_db_uri</strong> method.</p> +you can use <strong>sync</strong> method. Two sync methods are available +currently: <strong>tgz</strong> and <strong>git</strong>.</p> <p>Note that before add any package you should add a category for it using <strong>add_category</strong>. Then packages can be added using <strong>add_package</strong>. PackageDB currently does not write changes automatically, so you should call <strong>write</strong> after changes are done. This is not relevant diff --git a/docs/developer_instructions.rst b/docs/developer_instructions.rst index f0667ea..f868d2f 100644 --- a/docs/developer_instructions.rst +++ b/docs/developer_instructions.rst @@ -242,7 +242,8 @@ to add categories and packages and to do queries on them. Usually you do not wan class. If you have a database that should be synced with another already generate database -you can redifine URI to be used for syncing using **get_real_db_uri** method. +you can use **sync** method. Two sync methods are available +currently: **tgz** and **git**. Note that before add any package you should add a category for it using **add_category**. Then packages can be added using **add_package**. PackageDB currently does not write changes diff --git a/docs/g-sorcery.8 b/docs/g-sorcery.8 index 6782c98..c2f5cbb 100644 --- a/docs/g-sorcery.8 +++ b/docs/g-sorcery.8 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH G-SORCERY 8 "2015-04-20" "0.2" "g-sorcery" +.TH G-SORCERY 8 "2015-04-20" "0.2.1" "g-sorcery" .SH NAME g-sorcery \- manage overlays for 3rd party software providers . diff --git a/docs/g-sorcery.8.rst b/docs/g-sorcery.8.rst index 1e9af4e..2ace6b0 100644 --- a/docs/g-sorcery.8.rst +++ b/docs/g-sorcery.8.rst @@ -11,7 +11,7 @@ manage overlays for 3rd party software providers by Brian Dolbec. Integration with layman based on work of Auke Booij. :Date: 2015-04-20 :Copyright: Copyright (c) 2013-2015 Jauhien Piatlicki, License: GPL-2 -:Version: 0.2 +:Version: 0.2.1 :Manual section: 8 :Manual group: g-sorcery diff --git a/docs/g-sorcery.cfg.8 b/docs/g-sorcery.cfg.8 index ea97ada..f9c02ce 100644 --- a/docs/g-sorcery.cfg.8 +++ b/docs/g-sorcery.cfg.8 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH G-SORCERY.CFG 8 "2015-04-20" "0.2" "g-sorcery" +.TH G-SORCERY.CFG 8 "2015-04-20" "0.2.1" "g-sorcery" .SH NAME g-sorcery.cfg \- custom settings for g-sorcery . diff --git a/docs/g-sorcery.cfg.8.rst b/docs/g-sorcery.cfg.8.rst index f9b5aac..bd85d08 100644 --- a/docs/g-sorcery.cfg.8.rst +++ b/docs/g-sorcery.cfg.8.rst @@ -11,7 +11,7 @@ custom settings for g-sorcery by Brian Dolbec. :Date: 2015-04-20 :Copyright: Copyright (c) 2013-2015 Jauhien Piatlicki, License: GPL-2 -:Version: 0.2 +:Version: 0.2.1 :Manual section: 8 :Manual group: g-sorcery diff --git a/g_sorcery/backend.py b/g_sorcery/backend.py index e606348..809ff18 100644 --- a/g_sorcery/backend.py +++ b/g_sorcery/backend.py @@ -4,10 +4,10 @@ """ backend.py ~~~~~~~~~~ - + base class for backends - - :copyright: (c) 2013 by Jauhien Piatlicki + + :copyright: (c) 2013-2015 by Jauhien Piatlicki :license: GPL-2, see LICENSE for more details. """ @@ -30,7 +30,7 @@ class Backend(object): Command format is as follows: g-backend [-o overlay_dir] [-r repository] command - + where command is one of the following: sync list @@ -38,11 +38,11 @@ class Backend(object): generate package_name generate-tree [-d --digest] install package_name [portage flags] - + If no overlay directory is given the default one from backend config is used. """ - - def __init__(self, package_db_generator_class, + + def __init__(self, package_db_generator_class, ebuild_g_with_digest_class, ebuild_g_without_digest_class, eclass_g_class, metadata_g_class, package_db_class=PackageDB, sync_db=False): @@ -74,7 +74,7 @@ class Backend(object): p_generate_tree = subparsers.add_parser('generate-tree') p_generate_tree.add_argument('-d', '--digest', action='store_true') p_generate_tree.set_defaults(func=self.generate_tree) - + p_install = subparsers.add_parser('install') p_install.add_argument('pkgname') p_install.add_argument('pkgmanager_flags', nargs=argparse.REMAINDER) @@ -150,7 +150,7 @@ class Backend(object): if repository: if not "repositories" in config: - self.logger.error("repository " + repository + + self.logger.error("repository " + repository + " specified, but there is no repositories entry in config") return -1 repositories = config["repositories"] @@ -161,11 +161,15 @@ class Backend(object): else: self.logger.error('no repository given\n') return -1 - + + try: + sync_method = repository_config["sync_method"] + except KeyError: + sync_method = "tgz" if self.sync_db: pkg_db = self.package_db_generator(backend_path, repository, common_config, repository_config, generate=False) - pkg_db.sync(repository_config["db_uri"]) + pkg_db.sync(repository_config["db_uri"], repository_config=repository_config, sync_method=sync_method) else: pkg_db = self.package_db_generator(backend_path, repository, common_config, repository_config) @@ -227,7 +231,7 @@ class Backend(object): except Exception as e: self.logger.error('dependency solving failed: ' + str(e) + '\n') return -1 - + eclasses = [] for package in dependencies: eclasses += pkg_db.get_package_description(package)['eclasses'] @@ -409,17 +413,17 @@ class Backend(object): try: versions = package_db.list_package_versions(pkg.category, pkg.package) - for version in versions: + for version in versions: solved_deps, unsolved_deps = self.solve_dependencies(package_db, Package(pkg.category, pkg.package, version), solved_deps, unsolved_deps) except InvalidKeyError: # ignore non existing packages continue - + solved_deps.add(package) unsolved_deps.remove(package) - + return (solved_deps, unsolved_deps) @@ -449,7 +453,7 @@ class Backend(object): for pkgname in pkgnames: directory = os.path.join(overlay, pkgname) fast_manifest(directory) - + def generate_tree(self, args, config, global_config): """ Generate entire overlay. @@ -509,7 +513,7 @@ class Backend(object): with open(os.path.join(overlay, 'metadata', 'layout.conf'), 'w') as f: f.write("repo-name = %s\n" % os.path.basename(overlay)) f.write("masters = %s\n" % masters_overlays) - + if args.digest: ebuild_g = self.ebuild_g_with_digest_class(pkg_db) else: @@ -566,6 +570,13 @@ class Backend(object): self.fast_digest(overlay, pkgnames) overlays.write(overlays_info) + try: + clean_db = config["repositories"][args.repository]["clean_db"] + except KeyError: + clean_db = False + if clean_db: + pkg_db.clean() + def install(self, args, config, global_config): """ Install a package. @@ -592,7 +603,7 @@ class Backend(object): package_manager_class = package_managers[package_manager] package_manager = package_manager_class() package_manager.install(args.pkgname, *args.pkgmanager_flags) - + def __call__(self, args, config, global_config): """ Execute a command diff --git a/g_sorcery/git_syncer/__init__.py b/g_sorcery/git_syncer/__init__.py new file mode 100644 index 0000000..4265cc3 --- /dev/null +++ b/g_sorcery/git_syncer/__init__.py @@ -0,0 +1 @@ +#!/usr/bin/env python diff --git a/g_sorcery/git_syncer/git_syncer.py b/g_sorcery/git_syncer/git_syncer.py new file mode 100644 index 0000000..0f6c58e --- /dev/null +++ b/g_sorcery/git_syncer/git_syncer.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +""" + git_syncer.py + ~~~~~~~~~~~~~ + + git sync helper + + :copyright: (c) 2015 by Jauhien Piatlicki + :license: GPL-2, see LICENSE for more details. +""" + +import os + +from g_sorcery.compatibility import TemporaryDirectory + +from g_sorcery.exceptions import SyncError +from g_sorcery.syncer import Syncer, SyncedData, TmpSyncedData + + +class GITSyncer(Syncer): + """ + Class used to sync with git repos. + """ + + def sync(self, db_uri, repository_config): + """ + Synchronize local directory with remote source. + + Args: + db_uri: URI for synchronization with remote source. + repository_config: repository config. + + Returns: + SyncedData object that gives access to the directory with data. + """ + if self.persistent_datadir is None: + tmp_dir = TemporaryDirectory() + path = os.path.join(tmp_dir.name, "remote") + else: + path = self.persistent_datadir + try: + branch = repository_config["branch"] + except KeyError: + branch = "master" + + if os.path.exists(path): + #TODO: allow changing of remotes/branches + self.pull(path) + else: + self.clone(db_uri, branch, path) + + if self.persistent_datadir is None: + return TmpSyncedData(path, tmp_dir) + else: + return SyncedData(path) + + + def clone(self, db_uri, branch, path): + if os.system("git clone --depth 1 --branch " + branch + " " + db_uri + " " + path): + raise SyncError("sync failed (clonning): " + db_uri) + + + def pull(self, path): + if os.system("cd " + path + " && git pull"): + raise SyncError("sync failed (pulling): " + path) diff --git a/g_sorcery/package_db.py b/g_sorcery/package_db.py index a88474d..8daa665 100644 --- a/g_sorcery/package_db.py +++ b/g_sorcery/package_db.py @@ -11,18 +11,18 @@ :license: GPL-2, see LICENSE for more details. """ -import glob import os import portage -from .compatibility import basestring, py2k, TemporaryDirectory +from .compatibility import basestring, py2k from .db_layout import DBLayout, JSON_FILE_SUFFIX, SUPPORTED_DB_LAYOUTS, SUPPORTED_FILE_FORMATS from .exceptions import DBError, DBLayoutError, DBStructureError, InvalidKeyError, SyncError -from .fileutils import FileJSON, load_remote_file, copy_all, wget +from .fileutils import FileJSON, load_remote_file, copy_all from .g_collections import Package from .logger import Logger +from .syncer import SUPPORTED_SYNCERS SUPPORTED_DB_STRUCTURES=[0, 1] @@ -137,6 +137,7 @@ class PackageDB(object): def __init__(self, directory, + persistent_datadir = None, preferred_layout_version=1, preferred_db_version=1, preferred_category_format=JSON_FILE_SUFFIX): @@ -157,6 +158,11 @@ class PackageDB(object): self.logger = Logger() self.directory = os.path.abspath(directory) + + self.persistent_datadir = persistent_datadir + if self.persistent_datadir is not None: + self.persistent_datadir = os.path.abspath(self.persistent_datadir) + self.preferred_layout_version = preferred_layout_version self.preferred_db_version = preferred_db_version self.preferred_category_format = preferred_category_format @@ -176,24 +182,30 @@ class PackageDB(object): self.categories = {} - def sync(self, db_uri): + def sync(self, db_uri, repository_config = None, sync_method="tgz"): """ Synchronize local database with remote database. Args: db_uri: URI for synchronization with remote database. - """ - real_db_uri = self.get_real_db_uri(db_uri) - download_dir = TemporaryDirectory() - if wget(real_db_uri, download_dir.name): - raise SyncError('sync failed: ' + real_db_uri) - - temp_dir = TemporaryDirectory() - for f_name in glob.iglob(os.path.join(download_dir.name, '*.tar.gz')): - self.logger.info("unpacking " + f_name) - os.system("tar -xvzf " + f_name + " -C " + temp_dir.name) + repository_config: repository config. + sync_method: sync method (tgz or git). + """ + if repository_config is None: + repository_config = {} + + try: + syncer_cls = SUPPORTED_SYNCERS[sync_method] + except KeyError: + raise SyncError('unsupported sync method: ' + sync_method) + if self.persistent_datadir is not None: + remotedb_dir = os.path.join(self.persistent_datadir, 'remote') + else: + remotedb_dir = None + syncer = syncer_cls(remotedb_dir) + synced_data = syncer.sync(db_uri, repository_config) - tempdb_dir = os.path.join(temp_dir.name, os.listdir(temp_dir.name)[0]) + tempdb_dir = synced_data.get_path() tempdb = PackageDB(tempdb_dir) tempdb.db_layout.check_manifest() @@ -204,19 +216,7 @@ class PackageDB(object): self.db_layout.check_manifest() - del download_dir - del temp_dir - - - def get_real_db_uri(self, db_uri): - """ - Convert self.db_uri to URI where remote database can be - fetched from. - - Returns: - URI of remote database file. - """ - return db_uri + del synced_data def clean(self): @@ -538,10 +538,12 @@ class DBGenerator(object): Package database. """ db_path = os.path.join(directory, repository, "db") + persistent_datadir = os.path.join(directory, repository, "persistent") pkg_db = self.package_db_class(db_path, preferred_layout_version=self.preferred_layout_version, preferred_db_version=self.preferred_db_version, - preferred_category_format=self.preferred_category_format) + preferred_category_format=self.preferred_category_format, + persistent_datadir=persistent_datadir) config_f = FileJSON(os.path.join(directory, repository), "config.json", []) diff --git a/g_sorcery/syncer.py b/g_sorcery/syncer.py new file mode 100644 index 0000000..beeffd4 --- /dev/null +++ b/g_sorcery/syncer.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +""" + syncer.py + ~~~~~~~~~ + + sync helper + + :copyright: (c) 2013-2015 by Jauhien Piatlicki + :license: GPL-2, see LICENSE for more details. +""" + +import glob +import os + +from .compatibility import TemporaryDirectory + +from .exceptions import SyncError +from .fileutils import wget + + +class SyncedData(object): + """ + Synced data. + + Directory with sync data is guaranted to exist only as long as this + object does. + """ + def __init__(self, directory): + self.directory = os.path.abspath(directory) + + def get_path(self): + return self.directory + + +class TmpSyncedData(SyncedData): + """ + Synced data that lives in a temporary directory. + """ + + def __init__(self, directory, tmpdirobj): + super(TmpSyncedData, self).__init__(directory) + self.tmpdirobj = tmpdirobj + + +class Syncer(object): + """ + Class used to sync data with remote source. + """ + + def __init__(self, persistent_datadir): + self.persistent_datadir = persistent_datadir + + def sync(self, db_uri, repository_config): + """ + Synchronize local directory with remote source. + + Args: + db_uri: URI for synchronization with remote source. + repository_config: repository config. + + Returns: + SyncedData object that gives access to the directory with data. + """ + raise NotImplementedError + + +class TGZSyncer(Syncer): + """ + Class used to download and unpack tarballs. + """ + + def sync(self, db_uri, repository_config): + """ + Synchronize local directory with remote source. + + Args: + db_uri: URI for synchronization with remote source. + repository_config: repository config. + + Returns: + SyncedData object that gives access to the directory with data. + """ + download_dir = TemporaryDirectory() + if wget(db_uri, download_dir.name): + raise SyncError('sync failed: ' + db_uri) + + tmp_dir = TemporaryDirectory() + for f_name in glob.iglob(os.path.join(download_dir.name, '*.tar.gz')): + if os.system("tar -xvzf " + f_name + " -C " + tmp_dir.name): + raise SyncError('sync failed (unpacking)') + + tmp_path = os.path.join(tmp_dir.name, os.listdir(tmp_dir.name)[0]) + del download_dir + return TmpSyncedData(tmp_path, tmp_dir) + + +SUPPORTED_SYNCERS = {"tgz": TGZSyncer} + +# git_syncer module is optional, we should check if it is installed +try: + from .git_syncer.git_syncer import GITSyncer + SUPPORTED_SYNCERS["git"] = GITSyncer + +except ImportError as e: + pass @@ -18,7 +18,7 @@ import os from distutils.core import setup -SELECTABLE = {'bson': 'file_bson'} +SELECTABLE = {'bson': 'file_bson', 'git': 'git_syncer'} use_defaults = ' '.join(list(SELECTABLE)) USE = os.environ.get("USE", use_defaults).split() @@ -29,7 +29,7 @@ for mod in SELECTABLE: optional_modules.append('g_sorcery.%s' % SELECTABLE[mod]) setup(name = 'g-sorcery', - version = '0.2', + version = '0.2.1', description = 'framework for automated ebuild generators', author = 'Jauhien Piatlicki', author_email = 'jauhien@gentoo.org', diff --git a/tests/test_PackageDB.py b/tests/test_PackageDB.py index 179cc32..152c605 100644 --- a/tests/test_PackageDB.py +++ b/tests/test_PackageDB.py @@ -34,17 +34,12 @@ except ImportError as e: pass -class TestDB(PackageDB): - def get_real_db_uri(self, db_uri): - return db_uri + "/dummy.tar.gz" - - class TestPackageDB(BaseTest): def test_functionality(self): port = 8080 for fmt in SUPPORTED_FILE_FORMATS: - sync_address = "127.0.0.1:" + str(port) + sync_address = "127.0.0.1:" + str(port) + "/dummy.tar.gz" orig_tempdir = TemporaryDirectory() orig_path = os.path.join(orig_tempdir.name, "db") os.makedirs(orig_path) @@ -70,7 +65,7 @@ class TestPackageDB(BaseTest): os.system("echo invalid >> " + orig_tempdir.name + "/db/app-test1/packages." + fmt) os.system("cd " + orig_tempdir.name + " && tar cvzf dummy.tar.gz db") - test_db = TestDB(self.tempdir.name) + test_db = PackageDB(self.tempdir.name) self.assertRaises(SyncError, test_db.sync, sync_address) srv = Server(orig_tempdir.name, port=port) |