From 1ddc073811b7b69aab22cd65990bbca8e7104bef Mon Sep 17 00:00:00 2001 From: fuzzyray Date: Fri, 12 Mar 2010 21:44:29 +0000 Subject: Update to genscripts rev 382. This has more fixes for py3k and the modular rewrite of eclean. svn path=/trunk/gentoolkit/; revision=755 --- pym/gentoolkit/analyse/__init__.py | 6 +- pym/gentoolkit/analyse/analyse.py | 77 ++- pym/gentoolkit/analyse/base.py | 8 +- pym/gentoolkit/analyse/lib.py | 57 +- pym/gentoolkit/analyse/rebuild.py | 56 +- pym/gentoolkit/base.py | 7 +- pym/gentoolkit/cpv.py | 150 ++++- pym/gentoolkit/eclean/__init__.py | 4 + pym/gentoolkit/eclean/clean.py | 149 +++++ pym/gentoolkit/eclean/cli.py | 500 ++++++++++++++ pym/gentoolkit/eclean/exclude.py | 262 ++++++++ pym/gentoolkit/eclean/output.py | 179 +++++ pym/gentoolkit/eclean/pkgindex.py | 89 +++ pym/gentoolkit/eclean/search.py | 520 +++++++++++++++ pym/gentoolkit/equery/__init__.py | 12 +- pym/gentoolkit/equery/changes.py | 3 +- pym/gentoolkit/equery/list_.py | 26 +- pym/gentoolkit/equery/meta.py | 48 +- pym/gentoolkit/equery/uses.py | 16 +- pym/gentoolkit/errors.py | 37 +- pym/gentoolkit/helpers.py | 17 +- pym/gentoolkit/keyword.py | 8 +- pym/gentoolkit/metadata.py | 9 +- pym/gentoolkit/package.py | 45 +- pym/gentoolkit/query.py | 21 +- pym/gentoolkit/test/eclean/Packages | 1017 +++++++++++++++++++++++++++++ pym/gentoolkit/test/eclean/__init__.py | 6 + pym/gentoolkit/test/eclean/creator.py | 243 +++++++ pym/gentoolkit/test/eclean/test_clean.py | 149 +++++ pym/gentoolkit/test/eclean/test_search.py | 100 +++ pym/gentoolkit/test/test_helpers.py | 21 - pym/gentoolkit/test/test_keyword.py | 14 +- pym/gentoolkit/test/test_query.py | 111 ++++ 33 files changed, 3730 insertions(+), 237 deletions(-) create mode 100644 pym/gentoolkit/eclean/__init__.py create mode 100644 pym/gentoolkit/eclean/clean.py create mode 100644 pym/gentoolkit/eclean/cli.py create mode 100644 pym/gentoolkit/eclean/exclude.py create mode 100644 pym/gentoolkit/eclean/output.py create mode 100644 pym/gentoolkit/eclean/pkgindex.py create mode 100644 pym/gentoolkit/eclean/search.py create mode 100644 pym/gentoolkit/test/eclean/Packages create mode 100644 pym/gentoolkit/test/eclean/__init__.py create mode 100644 pym/gentoolkit/test/eclean/creator.py create mode 100644 pym/gentoolkit/test/eclean/test_clean.py create mode 100644 pym/gentoolkit/test/eclean/test_search.py create mode 100644 pym/gentoolkit/test/test_query.py (limited to 'pym') diff --git a/pym/gentoolkit/analyse/__init__.py b/pym/gentoolkit/analyse/__init__.py index 7a5fbec..82625b6 100644 --- a/pym/gentoolkit/analyse/__init__.py +++ b/pym/gentoolkit/analyse/__init__.py @@ -19,7 +19,7 @@ __version__ = "svn" __productname__ = "analyse" __authors__ = ( 'Brian Dolbec, ' - + ) # make an exportable copy of the info for help output @@ -29,7 +29,7 @@ MODULE_INFO = { "__version__": __version__, "__productname__": __productname__, "__authors__": __authors__ - + } import errno @@ -54,7 +54,7 @@ NAME_MAP = { } FORMATTED_OPTIONS = ( - (" (a)nalyse", + (" (a)nalyse", "analyses the installed PKG database USE flag or keyword useage"), (" (r)ebuild", "analyses the Installed PKG database and generates files suitable"), diff --git a/pym/gentoolkit/analyse/analyse.py b/pym/gentoolkit/analyse/analyse.py index 6797510..2162324 100644 --- a/pym/gentoolkit/analyse/analyse.py +++ b/pym/gentoolkit/analyse/analyse.py @@ -20,6 +20,7 @@ from gentoolkit.analyse.lib import (get_installed_use, get_iuse, abs_flag, abs_list, get_all_cpv_use, get_flags, FlagAnalyzer, KeywordAnalyser) from gentoolkit.analyse.output import nl, AnalysisPrinter from gentoolkit.package import Package +from gentoolkit.helpers import get_installed_cpvs import portage @@ -35,15 +36,31 @@ def gather_flags_info( _get_used=get_installed_use ): """Analyse the installed pkgs USE flags for frequency of use - + + @type cpvs: list @param cpvs: optional list of [cat/pkg-ver,...] to analyse or defaults to entire installed pkg db + @type: system_flags: list + @param system_flags: the current default USE flags as defined + by portage.settings["USE"].split() + @type include_unset: bool + @param include_unset: controls the inclusion of unset USE flags in the report. + @type target: string + @param target: the environment variable being analysed + one of ["USE", "PKGUSE"] + @type _get_flags: function + @param _get_flags: ovride-able for testing, + defaults to gentoolkit.analyse.lib.get_flags + @param _get_used: ovride-able for testing, + defaults to gentoolkit.analyse.lib.get_installed_use @rtype dict. {flag:{"+":[cat/pkg-ver,...], "-":[cat/pkg-ver,...], "unset":[]} """ if cpvs is None: cpvs = VARDB.cpv_all() # pass them in to override for tests flags = FlagAnalyzer(system_flags, + filter_defaults=False, + target=target, _get_flags=_get_flags, _get_used=get_installed_use ) @@ -87,7 +104,7 @@ def gather_keywords_info( analyser = None ): """Analyse the installed pkgs 'keywords' for frequency of use - + @param cpvs: optional list of [cat/pkg-ver,...] to analyse or defaults to entire installed pkg db @param system_keywords: list of the system keywords @@ -139,7 +156,7 @@ class Analyse(ModuleBase): """Installed db analysis tool to query the installed databse and produce/output stats for USE flags or keywords/mask. The 'rebuild' action output is in the form suitable for file type output - to create a new package.use, package.keywords, package.unmask + to create a new package.use, package.keywords, package.unmask type files in the event of needing to rebuild the /etc/portage/* user configs """ @@ -153,7 +170,7 @@ class Analyse(ModuleBase): "verbose": False, "quiet": False, 'prefix': False, - 'portage': False + 'portage': True } self.module_opts = { "-f": ("flags", "boolean", True), @@ -166,8 +183,8 @@ class Analyse(ModuleBase): "--verbose": ("verbose", "boolean", True), "-p": ("prefix", "boolean", True), "--prefix": ("prefix", "boolean", True), - "-G": ("portage", "boolean", True), - "--portage": ("portage", "boolean", True), + "-G": ("portage", "boolean", False), + "--portage": ("portage", "boolean", False), } self.formatted_options = [ (" -h, --help", "Outputs this useage message"), @@ -177,17 +194,17 @@ class Analyse(ModuleBase): (" -u, --unset", "Additionally include any unset USE flags and the packages"), ("", "that could use them"), - (" -v, --verbose", + (" -v, --verbose", "Used in the analyse action to output more detailed information"), - (" -p, --prefix", + (" -p, --prefix", "Used for testing purposes only, runs report using " + "a prefix keyword and 'prefix' USE flag"), - (" -G, --portage", - "Use portage directly instead of gentoolkit's Package " + - "object for some operations. Usually a little faster."), + #(" -G, --portage", + #"Use portage directly instead of gentoolkit's Package " + + #"object for some operations. Usually a little faster."), ] self.formatted_args = [ - (" use", + (" use", "causes the action to analyse the installed packages USE flags"), (" pkguse", "causes the action to analyse the installed packages PKGUSE flags"), @@ -197,15 +214,21 @@ class Analyse(ModuleBase): "causes the action to analyse the installed packages keywords"), ] self.short_opts = "huvpG" - self.long_opts = ("help", "unset", "verbose", "prefix", "portage") + self.long_opts = ("help", "unset", "verbose", "prefix") #, "portage") self.need_queries = True self.arg_spec = "Target" self.arg_options = ['use', 'pkguse','keywords'] self.arg_option = False + self.warning = ( + " CAUTION", + "This is beta software and some features/options are incomplete,", + "some features may change in future releases includig its name.", + "Feedback will be appreciated, http://bugs.gentoo.org") + def run(self, input_args, quiet=False): """runs the module - + @param input_args: input arguments to be parsed """ query = self.main_setup(input_args) @@ -216,10 +239,10 @@ class Analyse(ModuleBase): self.analyse_keywords() def analyse_flags(self, target): - """This will scan the installed packages db and analyse the + """This will scan the installed packages db and analyse the USE flags used for installation and produce a report on how they were used. - + @type target: string @param target: the target to be analysed, one of ["use", "pkguse"] """ @@ -227,17 +250,18 @@ class Analyse(ModuleBase): self.printer = AnalysisPrinter("use", self.options["verbose"], system_use) if self.options["verbose"]: cpvs = VARDB.cpv_all() + #cpvs = get_installed_cpvs() #print "Total number of installed ebuilds =", len(cpvs) flag_users = gather_flags_info(cpvs, system_use, self.options["unset"], target=target.upper(), use_portage=self.options['portage']) else: - flag_users = gather_flags_info(system_flags=system_use, + cpvs = get_installed_cpvs() + flag_users = gather_flags_info(cpvs, system_flags=system_use, include_unset=self.options["unset"], target=target.upper(), use_portage=self.options['portage']) #print flag_users - flag_keys = list(flag_users.keys()) - flag_keys.sort() + flag_keys = sorted(flag_users) if self.options["verbose"]: print(" Flag System #pkgs cat/pkg-ver") blankline = nl @@ -261,12 +285,12 @@ class Analyse(ModuleBase): print("===================================================") print("Total number of flags in report =", pp.output.red(str(len(flag_keys)))) if self.options["verbose"]: - print("Total number of installed ebuilds =", pp.output.red(str(len(cpvs)))) + print("Total number of installed ebuilds =", pp.output.red(str(len([x for x in cpvs])))) print() def analyse_keywords(self, keywords=None): - """This will scan the installed packages db and analyse the + """This will scan the installed packages db and analyse the keywords used for installation and produce a report on them. """ print() @@ -295,7 +319,7 @@ class Analyse(ModuleBase): cpvs = VARDB.cpv_all() #print "Total number of installed ebuilds =", len(cpvs) keyword_users = gather_keywords_info( - cpvs=cpvs, + cpvs=cpvs, system_keywords=system_keywords, use_portage=self.options['portage'], keywords=keywords, analyser = self.analyser @@ -305,13 +329,12 @@ class Analyse(ModuleBase): keyword_users = gather_keywords_info( system_keywords=system_keywords, use_portage=self.options['portage'], - keywords=keywords, + keywords=keywords, analyser = self.analyser ) blankline = lambda: None #print keyword_users - keyword_keys = list(keyword_users.keys()) - keyword_keys.sort() + keyword_keys = sorted(keyword_users) if self.options["verbose"]: print(" Keyword System #pkgs cat/pkg-ver") elif not self.options['quiet']: @@ -344,9 +367,9 @@ class Analyse(ModuleBase): def main(input_args): - """Common starting method by the analyse master + """Common starting method by the analyse master unless all modules are converted to this class method. - + @param input_args: input args as supplied by equery master module. """ query_module = Analyse() diff --git a/pym/gentoolkit/analyse/base.py b/pym/gentoolkit/analyse/base.py index bb0a8bc..d45ccc6 100644 --- a/pym/gentoolkit/analyse/base.py +++ b/pym/gentoolkit/analyse/base.py @@ -28,7 +28,7 @@ from gentoolkit.base import mod_usage class ModuleBase(object): """Analyse base module class to parse module options print module help, etc..""" - + def __init__(self): self.module_name = None self.options = {} @@ -36,7 +36,7 @@ class ModuleBase(object): self.short_opts = None self.long_opts = None self.module_opts = {} - self.depwarning = None + self.warning = None self.need_queries = True @@ -51,9 +51,9 @@ class ModuleBase(object): print() print(__doc__.strip()) print() - if self.depwarning: + if self.warning: print() - for line in self.depwarning: + for line in self.warning: sys.stderr.write(pp.warn(line)) print() print(mod_usage(mod_name=self.module_name, arg=self.arg_spec, optional=self.arg_option)) diff --git a/pym/gentoolkit/analyse/lib.py b/pym/gentoolkit/analyse/lib.py index 17e2118..a32d13b 100644 --- a/pym/gentoolkit/analyse/lib.py +++ b/pym/gentoolkit/analyse/lib.py @@ -20,7 +20,7 @@ import portage def get_installed_use(cpv, use="USE"): """Gets the installed USE flags from the VARDB - + @type: cpv: string @param cpv: cat/pkg-ver @type use: string @@ -33,7 +33,7 @@ def get_installed_use(cpv, use="USE"): def get_iuse(cpv): """Gets the current IUSE flags from the tree - + @type: cpv: string @param cpv: cat/pkg-ver @rtype list @@ -47,7 +47,7 @@ def get_iuse(cpv): def abs_flag(flag): """Absolute value function for a USE flag - + @type flag: string @param flag: the use flag to absolute. @rtype: string @@ -61,7 +61,7 @@ def abs_flag(flag): def abs_list(the_list): """Absolute value function for a USE flag list - + @type the_list: list @param the_list: the use flags to absolute. @rtype: list @@ -76,7 +76,7 @@ def abs_list(the_list): def filter_flags(use, use_expand_hidden, usemasked, useforced): """Filter function to remove hidden or otherwise not normally visible USE flags from a list. - + @type use: list @param use: the USE flag list to be filtered. @type use_expand_hidden: list @@ -110,7 +110,7 @@ def filter_flags(use, use_expand_hidden, usemasked, useforced): def get_all_cpv_use(cpv): """Uses portage to determine final USE flags and settings for an emerge - + @type cpv: string @param cpv: eg cat/pkg-ver @rtype: lists @@ -137,7 +137,7 @@ def get_all_cpv_use(cpv): def get_flags(cpv, final_setting=False): """Retrieves all information needed to filter out hidded, masked, etc. USE flags for a given package. - + @type cpv: string @param cpv: eg. cat/pkg-ver @type final_setting: boolean @@ -158,7 +158,7 @@ class FlagAnalyzer(object): """Specialty functions for analysing an installed package's USE flags. Can be used for single or mulitple use without needing to be reset unless the system USE flags are changed. - + @type system: list or set @param system: the default system USE flags. @type _get_flags: function @@ -168,17 +168,21 @@ class FlagAnalyzer(object): """ def __init__(self, system, + filter_defaults=False, + target="USE", _get_flags=get_flags, _get_used=get_installed_use ): self.get_flags = _get_flags self.get_used = _get_used + self.filter_defaults = filter_defaults + self.target = target self.reset(system) def reset(self, system): """Resets the internal system USE flags and use_expand variables to the new setting. The use_expand variable is handled internally. - + @type system: list or set @param system: the default system USE flags. """ @@ -189,19 +193,19 @@ class FlagAnalyzer(object): """Gets all relavent USE flag info for a cpv and breaks them down into 3 sets, plus (package.use enabled), minus ( package.use disabled), unset. - + @param cpv: string. 'cat/pkg-ver' @rtype tuple of sets @return (plus, minus, unset) sets of USE flags """ - installed = set(self.get_used(cpv, "USE")) + installed = set(self.get_used(cpv, self.target)) iuse = set(abs_list(self.get_flags(cpv))) return self._analyse(installed, iuse) def _analyse(self, installed, iuse): """Analyses the supplied info and returns the flag settings that differ from the defaults - + @type installed: set @param installed: the installed with use flags @type iuse: set @@ -209,7 +213,10 @@ class FlagAnalyzer(object): """ defaults = self.system.intersection(iuse) usedflags = iuse.intersection(set(installed)) - plus = usedflags.difference(defaults) + if self.filter_defaults: + plus = usedflags.difference(defaults) + else: + plus = usedflags minus = defaults.difference(usedflags) unset = iuse.difference(defaults, plus, minus) cleaned = self.remove_expanding(unset) @@ -219,7 +226,7 @@ class FlagAnalyzer(object): """Gets all relevent USE flag info for a pkg and breaks them down into 3 sets, plus (package.use enabled), minus ( package.use disabled), unset. - + @param pkg: gentoolkit.package.Package object @rtype tuple of sets @return (plus, minus, unset) sets of USE flags @@ -229,7 +236,9 @@ class FlagAnalyzer(object): return self._analyse(installed, iuse) def pkg_used(self, pkg): - return pkg.use().split() + if self.target == "USE": + return pkg.use().split() + return pkg.environment(self.target).split() def pkg_flags(self, pkg): final_use, use_expand_hidden, usemasked, useforced = \ @@ -246,7 +255,7 @@ class FlagAnalyzer(object): def remove_expanding(self, flags): """Remove unwanted USE_EXPAND flags from unset IUSE sets - + @param flags: short list or set of USE flags @rtype set @return USE flags @@ -264,12 +273,12 @@ class FlagAnalyzer(object): class KeywordAnalyser(object): """Specialty functions for analysing the installed package db for keyword useage and the packages that used them. - + Note: should be initialized with the internal set_order() before use. See internal set_order() for more details. This class of functions can be used for single cpv checks or used repeatedly for an entire package db. - + @type arch: string @param arch: the system ARCH setting @type accept_keywords: list @@ -302,9 +311,9 @@ class KeywordAnalyser(object): self.mismatched = [] def determine_keyword(self, keywords, used, cpv): - """Determine the keyword from the installed USE flags and + """Determine the keyword from the installed USE flags and the KEYWORDS that was used to install a package. - + @param keywords: list of keywords available to install a pkg @param used: list of USE flalgs recorded for the installed pkg @rtype: string @@ -396,7 +405,7 @@ class KeywordAnalyser(object): def get_inst_keyword_cpv(self, cpv): """Determines the installed with keyword for cpv - + @type cpv: string @param cpv: an installed CAT/PKG-VER @rtype: string @@ -409,7 +418,7 @@ class KeywordAnalyser(object): def get_inst_keyword_pkg(self, pkg): """Determines the installed with keyword for cpv - + @param pkg: gentoolkit.package.Package object @rtype: string @returns a keyword determined to have been used to install cpv @@ -447,11 +456,11 @@ class KeywordAnalyser(object): def set_order(self, used): """Used to set the parsing order to determine a keyword used for installation. - + This is needed due to the way prefix arch's and keywords work with portage. It looks for the 'prefix' flag. A positive result sets it to the prefix order and keyword. - + @type used: list @param used: a list of pkg USE flags or the system USE flags""" if 'prefix' in used: diff --git a/pym/gentoolkit/analyse/rebuild.py b/pym/gentoolkit/analyse/rebuild.py index 31c00a1..c7cf1e5 100644 --- a/pym/gentoolkit/analyse/rebuild.py +++ b/pym/gentoolkit/analyse/rebuild.py @@ -9,16 +9,12 @@ """Provides a rebuild file of USE flags or keywords used and by what packages according to the Installed package database""" -from __future__ import print_function - -# Move to Imports section after Python 2.6 is stable +from __future__ import print_function import sys -from portage import os - import gentoolkit from gentoolkit.dbapi import PORTDB, VARDB from gentoolkit.analyse.base import ModuleBase @@ -28,6 +24,7 @@ from gentoolkit.analyse.lib import (get_installed_use, get_flags, from gentoolkit.analyse.output import RebuildPrinter import portage +from portage import os def cpv_all_diff_use( @@ -37,12 +34,31 @@ def cpv_all_diff_use( _get_flags=get_flags, _get_used=get_installed_use ): + """Data gathering and analysis function determines + the difference between the current default USE flag settings + and the currently installed pkgs recorded USE flag settings + + @type cpvs: list + @param cpvs: optional list of [cat/pkg-ver,...] to analyse or + defaults to entire installed pkg db + @type: system_flags: list + @param system_flags: the current default USE flags as defined + by portage.settings["USE"].split() + @type _get_flags: function + @param _get_flags: ovride-able for testing, + defaults to gentoolkit.analyse.lib.get_flags + @param _get_used: ovride-able for testing, + defaults to gentoolkit.analyse.lib.get_installed_use + @rtype dict. {cpv:['flag1', '-flag2',...]} + """ if cpvs is None: cpvs = VARDB.cpv_all() cpvs.sort() data = {} # pass them in to override for tests flags = FlagAnalyzer(system_flags, + filter_defaults=True, + target="USE", _get_flags=_get_flags, _get_used=get_installed_use ) @@ -59,7 +75,7 @@ class Rebuild(ModuleBase): """Installed db analysis tool to query the installed databse and produce/output stats for USE flags or keywords/mask. The 'rebuild' action output is in the form suitable for file type output - to create a new package.use, package.keywords, package.unmask + to create a new package.use, package.keywords, package.unmask type files in the event of needing to rebuild the /etc/portage/* user configs """ @@ -92,7 +108,7 @@ class Rebuild(ModuleBase): (" ", "leading '=' and include the version") ] self.formatted_args = [ - (" use", + (" use", "causes the action to analyse the installed packages USE flags"), (" keywords", "causes the action to analyse the installed packages keywords"), @@ -106,12 +122,18 @@ class Rebuild(ModuleBase): self.arg_spec = "TargetSpec" self.arg_options = ['use', 'keywords', 'unmask'] self.arg_option = False + self.warning = ( + " CAUTION", + "This is beta software and some features/options are incomplete,", + "some features may change in future releases includig its name.", + "The file generated is saved in your home directory", + "Feedback will be appreciated, http://bugs.gentoo.org") def run(self, input_args, quiet=False): """runs the module - + @param input_args: input arguments to be parsed """ self.options['quiet'] = quiet @@ -121,8 +143,8 @@ class Rebuild(ModuleBase): self.rebuild_use() elif query in ["keywords"]: self.rebuild_keywords() - elif query in ["mask"]: - self.rebuild_mask() + elif query in ["unmask"]: + self.rebuild_unmask() def rebuild_use(self): @@ -140,11 +162,10 @@ class Rebuild(ModuleBase): pp.emph(" packages that need entries"))) #print pp.emph(" package.use to maintain their current setting") if pkgs: - pkg_keys = list(pkgs.keys()) - pkg_keys.sort() + pkg_keys = sorted(pkgs) #print len(pkgs) if self.options["pretend"] and not self.options["quiet"]: - print() + print() print(pp.globaloption( " -- These are the installed packages & flags " + "that were detected")) @@ -183,29 +204,28 @@ class Rebuild(ModuleBase): print("Module action not yet available") print() - def rebuild_mask(self): + def rebuild_unmask(self): print("Module action not yet available") print() def save_file(self, filepath, data): """Writes the data to the file determined by filepath - + @param filepath: string. eg. '/path/to/filename' @param data: list of lines to write to filepath """ if not self.options["quiet"]: print(' - Saving file: %s' %filepath) - #print "Just kidding :) I haven't codded it yet" with open(filepath, "w") as output: output.write('\n'.join(data)) print(" - Done") def main(input_args): - """Common starting method by the analyse master + """Common starting method by the analyse master unless all modules are converted to this class method. - + @param input_args: input args as supplied by equery master module. """ query_module = Rebuild() diff --git a/pym/gentoolkit/base.py b/pym/gentoolkit/base.py index 9819390..29d3279 100644 --- a/pym/gentoolkit/base.py +++ b/pym/gentoolkit/base.py @@ -1,12 +1,11 @@ -# Copyright(c) 2009, Gentoo Foundation +#!/usr/bin/python # -# Copyright 2010 Brian Dolbec -# Copyright(c) 2010, Gentoo Foundation +# Copyright(c) 2009 - 2010, Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 # # $Header: $ -"""Analyse Base Module class to hold common module operation functions +"""Gentoolkit Base Module class to hold common module operation functions """ from __future__ import print_function diff --git a/pym/gentoolkit/cpv.py b/pym/gentoolkit/cpv.py index 9f2c303..663afea 100644 --- a/pym/gentoolkit/cpv.py +++ b/pym/gentoolkit/cpv.py @@ -1,6 +1,8 @@ #!/usr/bin/python # -# Copyright(c) 2009, Gentoo Foundation +# Copyright(c) 2005 Jason Stubbs +# Copyright(c) 2005-2006 Brian Harring +# Copyright(c) 2009-2010 Gentoo Foundation # # Licensed under the GNU General Public License, v2 # @@ -18,10 +20,21 @@ __all__ = ( # Imports # ======= +import re + from portage.versions import catpkgsplit, vercmp, pkgcmp from gentoolkit import errors +# ======= +# Globals +# ======= + +isvalid_version_re = re.compile("^(?:cvs\\.)?(?:\\d+)(?:\\.\\d+)*[a-z]?" + "(?:_(p(?:re)?|beta|alpha|rc)\\d*)*$") +isvalid_cat_re = re.compile("^(?:[a-zA-Z0-9][-a-zA-Z0-9+._]*(?:/(?!$))?)+$") +_pkg_re = re.compile("^[a-zA-Z0-9+_]+$") + # ======= # Classes # ======= @@ -43,25 +56,63 @@ class CPV(object): True """ - def __init__(self, cpv): + def __init__(self, cpv, validate=False): self.cpv = cpv - - values = split_cpv(cpv) - self.category = values[0] - self.name = values[1] - self.version = values[2] - self.revision = values[3] - del values - - if not self.name: + self._category = None + self._name = None + self._version = None + self._revision = None + self._cp = None + self._fullversion = None + + self.validate = validate + if validate and not self.name: raise errors.GentoolkitInvalidCPV(cpv) - sep = '/' if self.category else '' - self.cp = sep.join((self.category, self.name)) - - sep = '-' if self.revision else '' - self.fullversion = sep.join((self.version, self.revision)) - del sep + @property + def category(self): + if self._category is None: + self._set_cpv_chunks() + return self._category + + @property + def name(self): + if self._name is None: + self._set_cpv_chunks() + return self._name + + @property + def version(self): + if self._version is None: + self._set_cpv_chunks() + return self._version + + @property + def revision(self): + if self._revision is None: + self._set_cpv_chunks() + return self._revision + + @property + def cp(self): + if self._cp is None: + sep = '/' if self.category else '' + self._cp = sep.join((self.category, self.name)) + return self._cp + + @property + def fullversion(self): + if self._fullversion is None: + sep = '-' if self.revision else '' + self._fullversion = sep.join((self.version, self.revision)) + return self._fullversion + + def _set_cpv_chunks(self): + chunks = split_cpv(self.cpv, validate=self.validate) + self._category = chunks[0] + self._name = chunks[1] + self._version = chunks[2] + self._revision = chunks[3] def __eq__(self, other): if not isinstance(other, self.__class__): @@ -142,34 +193,63 @@ def compare_strs(pkg1, pkg2): return pkgcmp(pkg1[1:], pkg2[1:]) -def split_cpv(cpv): +def split_cpv(cpv, validate=True): """Split a cpv into category, name, version and revision. - Inlined from helpers because of circular imports. + Modified from pkgcore.ebuild.cpv - @todo: this function is slow and accepts some crazy things for cpv @type cpv: str - @param cpv: pkg, cat/pkg, pkg-ver, cat/pkg-ver, atom or regex + @param cpv: pkg, cat/pkg, pkg-ver, cat/pkg-ver @rtype: tuple @return: (category, pkg_name, version, revision) Each tuple element is a string or empty string (""). """ - result = catpkgsplit(cpv) + category = name = version = revision = '' + + try: + category, pkgver = cpv.rsplit("/", 1) + except ValueError: + pkgver = cpv + if validate and category and not isvalid_cat_re.match(category): + raise errors.GentoolkitInvalidCPV(cpv) + pkg_chunks = pkgver.split("-") + lpkg_chunks = len(pkg_chunks) + if lpkg_chunks == 1: + return (category, pkg_chunks[0], version, revision) + if isvalid_rev(pkg_chunks[-1]): + if lpkg_chunks < 3: + # needs at least ('pkg', 'ver', 'rev') + raise errors.GentoolkitInvalidCPV(cpv) + rev = pkg_chunks.pop(-1) + if rev: + revision = rev + + if validate and not isvalid_version_re.match(pkg_chunks[-1]): + raise errors.GentoolkitInvalidCPV(cpv) + version = pkg_chunks.pop(-1) + + if not isvalid_pkg_name(pkg_chunks): + raise errors.GentoolkitInvalidCPV(cpv) + name = '-'.join(pkg_chunks) + + return (category, name, version, revision) + + +def isvalid_pkg_name(chunks): + if not chunks[0]: + # this means a leading - + return False + mf = _pkg_re.match + if not all(not s or mf(s) for s in chunks): + return False + if chunks[-1].isdigit() or not chunks[-1]: + # not allowed. + return False + return True - if result: - result = list(result) - if result[0] == 'null': - result[0] = '' - if result[3] == 'r0': - result[3] = '' - else: - result = cpv.split("/") - if len(result) == 1: - result = ['', cpv, '', ''] - else: - result = result + ['', ''] - return tuple(result) +def isvalid_rev(s): + return s and s[0] == 'r' and s[1:].isdigit() # vim: set ts=4 sw=4 tw=79: diff --git a/pym/gentoolkit/eclean/__init__.py b/pym/gentoolkit/eclean/__init__.py new file mode 100644 index 0000000..edf9385 --- /dev/null +++ b/pym/gentoolkit/eclean/__init__.py @@ -0,0 +1,4 @@ +#!/usr/bin/python +# +# Copyright 2003-2010 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 diff --git a/pym/gentoolkit/eclean/clean.py b/pym/gentoolkit/eclean/clean.py new file mode 100644 index 0000000..9f6d597 --- /dev/null +++ b/pym/gentoolkit/eclean/clean.py @@ -0,0 +1,149 @@ +#!/usr/bin/python + +# Copyright 2003-2010 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + + +from __future__ import print_function + + +import sys + +from portage import os +import gentoolkit.pprinter as pp +from gentoolkit.eclean.pkgindex import PkgIndex + + +class CleanUp(object): + """Performs all cleaning actions to distfiles or package directories. + + @param controller: a progress output/user interaction controller function + which returns a Boolean to control file deletion + or bypassing/ignoring + """ + + def __init__(self, controller): + self.controller = controller + + def clean_dist(self, clean_dict): + """Calculate size of each entry for display, prompt user if needed, + delete files if approved and return the total size of files that + have been deleted. + + @param clean_dict: dictionary of {'display name':[list of files]} + + @rtype: int + @return: total size that was cleaned + """ + file_type = 'file' + clean_keys = self._sort_keys(clean_dict) + clean_size = 0 + # clean all entries one by one + for key in clean_keys: + clean_size += self._clean_files(clean_dict[key], key, file_type) + # return total size of deleted or to delete files + return clean_size + + def clean_pkgs(self, clean_dict, pkgdir): + """Calculate size of each entry for display, prompt user if needed, + delete files if approved and return the total size of files that + have been deleted. + + @param clean_dict: dictionary of {'display name':[list of files]} + @param metadata: package index of type portage.getbinpkg.PackageIndex() + @param pkgdir: path to the package directory to be cleaned + + @rtype: int + @return: total size that was cleaned + """ + file_type = 'binary package' + clean_keys = self._sort_keys(clean_dict) + clean_size = 0 + # clean all entries one by one + for key in clean_keys: + clean_size += self._clean_files(clean_dict[key], key, file_type) + + # run 'emaint --fix' here + if clean_size: + index_control = PkgIndex(self.controller) + # emaint is not yet importable so call it + # print a blank line here for separation + print() + clean_size += index_control.call_emaint() + # return total size of deleted or to delete files + return clean_size + + + def pretend_clean(self, clean_dict): + """Shortcut function that calculates total space savings + for the files in clean_dict. + + @param clean_dict: dictionary of {'display name':[list of files]} + @rtype: integer + @return: total size that would be cleaned + """ + file_type = 'file' + clean_keys = self._sort_keys(clean_dict) + clean_size = 0 + # tally all entries one by one + for key in clean_keys: + key_size = self._get_size(clean_dict[key]) + self.controller(key_size, key, clean_dict[key], file_type) + clean_size += key_size + return clean_size + + def _get_size(self, key): + """Determine the total size for an entry (may be several files).""" + key_size = 0 + for file_ in key: + #print file_ + # get total size for an entry (may be several files, and + # links don't count + # ...get its statinfo + try: + statinfo = os.stat(file_) + if statinfo.st_nlink == 1: + key_size += statinfo.st_size + except EnvironmentError as er: + print( pp.error( + "Could not get stat info for:" + file_), file=sys.stderr) + print( pp.error("Error: %s" %str(er)), file=sys.stderr) + return key_size + + def _sort_keys(self, clean_dict): + """Returns a list of sorted dictionary keys.""" + # sorting helps reading + clean_keys = sorted(clean_dict) + return clean_keys + + def _clean_files(self, files, key, file_type): + """File removal function.""" + clean_size = 0 + for file_ in files: + #print file_, type(file_) + # ...get its statinfo + try: + statinfo = os.stat(file_) + except EnvironmentError as er: + print( pp.error( + "Could not get stat info for:" + file_), file=sys.stderr) + print( pp.error( + "Error: %s" %str(er)), file=sys.stderr) + if self.controller(statinfo.st_size, key, file_, file_type): + # ... try to delete it. + try: + os.unlink(file_) + # only count size if successfully deleted and not a link + if statinfo.st_nlink == 1: + clean_size += statinfo.st_size + except EnvironmentError as er: + print( pp.error("Could not delete "+file_), file=sys.stderr) + print( pp.error("Error: %s" %str(er)), file=sys.stderr) + return clean_size + + + + + + + diff --git a/pym/gentoolkit/eclean/cli.py b/pym/gentoolkit/eclean/cli.py new file mode 100644 index 0000000..6a507ef --- /dev/null +++ b/pym/gentoolkit/eclean/cli.py @@ -0,0 +1,500 @@ +#!/usr/bin/python + +# Copyright 2003-2010 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + + +from __future__ import print_function + + +__author__ = "Thomas de Grenier de Latour (tgl), " + \ + "modular re-write by: Brian Dolbec (dol-sen)" +__email__ = "degrenier@easyconnect.fr, " + \ + "brian.dolbec@gmail.com" +__version__ = "svn" +__productname__ = "eclean" +__description__ = "A cleaning tool for Gentoo distfiles and binaries." + + +import sys +import re +import time +import getopt + +import portage +from portage import os +from portage.output import white, yellow, turquoise, green, teal, red + +import gentoolkit.pprinter as pp +from gentoolkit.eclean.search import (DistfilesSearch, + findPackages, port_settings, pkgdir) +from gentoolkit.eclean.exclude import (parseExcludeFile, + ParseExcludeFileException) +from gentoolkit.eclean.clean import CleanUp +from gentoolkit.eclean.output import OutputControl +#from gentoolkit.eclean.dbapi import Dbapi + +def printVersion(): + """Output the version info.""" + print( "%s (%s) - %s" \ + % (__productname__, __version__, __description__)) + print() + print("Author: %s <%s>" % (__author__,__email__)) + print("Copyright 2003-2009 Gentoo Foundation") + print("Distributed under the terms of the GNU General Public License v2") + + +def printUsage(_error=None, help=None): + """Print help message. May also print partial help to stderr if an + error from {'options','actions'} is specified.""" + + out = sys.stdout + if _error: + out = sys.stderr + if not _error in ('actions', 'global-options', \ + 'packages-options', 'distfiles-options', \ + 'merged-packages-options', 'merged-distfiles-options', \ + 'time', 'size'): + _error = None + if not _error and not help: help = 'all' + if _error == 'time': + print( pp.error("Wrong time specification"), file=out) + print( "Time specification should be an integer followed by a"+ + " single letter unit.", file=out) + print( "Available units are: y (years), m (months), w (weeks), "+ + "d (days) and h (hours).", file=out) + print( "For instance: \"1y\" is \"one year\", \"2w\" is \"two"+ + " weeks\", etc. ", file=out) + return + if _error == 'size': + print( pp.error("Wrong size specification"), file=out) + print( "Size specification should be an integer followed by a"+ + " single letter unit.", file=out) + print( "Available units are: G, M, K and B.", file=out) + print("For instance: \"10M\" is \"ten megabytes\", \"200K\" "+ + "is \"two hundreds kilobytes\", etc.", file=out) + return + if _error in ('global-options', 'packages-options', 'distfiles-options', \ + 'merged-packages-options', 'merged-distfiles-options',): + print( pp.error("Wrong option on command line."), file=out) + print( file=out) + elif _error == 'actions': + print( pp.error("Wrong or missing action name on command line."), file=out) + print( file=out) + print( white("Usage:"), file=out) + if _error in ('actions','global-options', 'packages-options', \ + 'distfiles-options') or help == 'all': + print( " "+turquoise(__productname__), + yellow("[global-option] ..."), + green(""), + yellow("[action-option] ..."), file=out) + if _error == 'merged-distfiles-options' or help in ('all','distfiles'): + print( " "+turquoise(__productname__+'-dist'), + yellow("[global-option, distfiles-option] ..."), file=out) + if _error == 'merged-packages-options' or help in ('all','packages'): + print( " "+turquoise(__productname__+'-pkg'), + yellow("[global-option, packages-option] ..."), file=out) + if _error in ('global-options', 'actions'): + print( " "+turquoise(__productname__), + yellow("[--help, --version]"), file=out) + if help == 'all': + print( " "+turquoise(__productname__+"(-dist,-pkg)"), + yellow("[--help, --version]"), file=out) + if _error == 'merged-packages-options' or help == 'packages': + print( " "+turquoise(__productname__+'-pkg'), + yellow("[--help, --version]"), file=out) + if _error == 'merged-distfiles-options' or help == 'distfiles': + print( " "+turquoise(__productname__+'-dist'), + yellow("[--help, --version]"), file=out) + print(file=out) + if _error in ('global-options', 'merged-packages-options', \ + 'merged-distfiles-options') or help: + print( "Available global", yellow("options")+":", file=out) + print( yellow(" -C, --nocolor")+ + " - turn off colors on output", file=out) + print( yellow(" -d, --destructive")+ + " - only keep the minimum for a reinstallation", file=out) + print( yellow(" -e, --exclude-file=")+ + " - path to the exclusion file", file=out) + print( yellow(" -i, --interactive")+ + " - ask confirmation before deletions", file=out) + print( yellow(" -n, --package-names")+ + " - protect all versions (when --destructive)", file=out) + print( yellow(" -p, --pretend")+ + " - only display what would be cleaned", file=out) + print( yellow(" -q, --quiet")+ + " - be as quiet as possible", file=out) + print( yellow(" -t, --time-limit=