From f0493e2db3e52bf919d8b4798332bbae7e31d50c Mon Sep 17 00:00:00 2001 From: dol-sen Date: Tue, 10 May 2011 23:46:34 -0700 Subject: analyse namespace change to enalyze and standardize the spelling to use the 'z' variant. --- bin/analyse | 48 ---- bin/enalyze | 48 ++++ man/analyse.1 | 210 ----------------- man/enalyze.1 | 210 +++++++++++++++++ pym/gentoolkit/analyse/__init__.py | 128 ----------- pym/gentoolkit/analyse/analyse.py | 448 ------------------------------------- pym/gentoolkit/analyse/base.py | 148 ------------ pym/gentoolkit/analyse/lib.py | 354 ----------------------------- pym/gentoolkit/analyse/output.py | 297 ------------------------ pym/gentoolkit/analyse/rebuild.py | 369 ------------------------------ pym/gentoolkit/enalyze/__init__.py | 128 +++++++++++ pym/gentoolkit/enalyze/analyze.py | 448 +++++++++++++++++++++++++++++++++++++ pym/gentoolkit/enalyze/base.py | 148 ++++++++++++ pym/gentoolkit/enalyze/lib.py | 354 +++++++++++++++++++++++++++++ pym/gentoolkit/enalyze/output.py | 297 ++++++++++++++++++++++++ pym/gentoolkit/enalyze/rebuild.py | 369 ++++++++++++++++++++++++++++++ 16 files changed, 2002 insertions(+), 2002 deletions(-) delete mode 100755 bin/analyse create mode 100755 bin/enalyze delete mode 100644 man/analyse.1 create mode 100644 man/enalyze.1 delete mode 100644 pym/gentoolkit/analyse/__init__.py delete mode 100644 pym/gentoolkit/analyse/analyse.py delete mode 100644 pym/gentoolkit/analyse/base.py delete mode 100644 pym/gentoolkit/analyse/lib.py delete mode 100644 pym/gentoolkit/analyse/output.py delete mode 100644 pym/gentoolkit/analyse/rebuild.py create mode 100644 pym/gentoolkit/enalyze/__init__.py create mode 100644 pym/gentoolkit/enalyze/analyze.py create mode 100644 pym/gentoolkit/enalyze/base.py create mode 100644 pym/gentoolkit/enalyze/lib.py create mode 100644 pym/gentoolkit/enalyze/output.py create mode 100644 pym/gentoolkit/enalyze/rebuild.py diff --git a/bin/analyse b/bin/analyse deleted file mode 100755 index a90410b..0000000 --- a/bin/analyse +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/python -# -# Copyright 2010 Brian Dolbec -# Copyright 2002-2010 Gentoo Technologies, Inc. -# Distributed under the terms of the GNU General Public License v2 or later -# -# $Header$ - -"""'analyse' is a flexible utility for Gentoo linux which can display various -information about installed packages, such as the USE flags used and the -packages that use them. It can also be used to help rebuild /etc/portage/package.* -files in the event of corruption, and possibly more. -""" - -from __future__ import print_function - -import sys -# This block ensures that ^C interrupts are handled quietly. -try: - import signal - - def exithandler(signum,frame): - signal.signal(signal.SIGINT, signal.SIG_IGN) - signal.signal(signal.SIGTERM, signal.SIG_IGN) - print() - sys.exit(1) - - signal.signal(signal.SIGINT, exithandler) - signal.signal(signal.SIGTERM, exithandler) - signal.signal(signal.SIGPIPE, signal.SIG_DFL) - -except KeyboardInterrupt: - print() - sys.exit(1) - -from gentoolkit import analyse, errors - -try: - analyse.main() -except errors.GentoolkitException as err: - if '--debug' in sys.argv: - raise - else: - from gentoolkit import pprinter as pp - sys.stderr.write(pp.error(str(err))) - print() - print("Add '--debug' to global options for traceback.") - sys.exit(1) diff --git a/bin/enalyze b/bin/enalyze new file mode 100755 index 0000000..e48c5b4 --- /dev/null +++ b/bin/enalyze @@ -0,0 +1,48 @@ +#!/usr/bin/python +# +# Copyright 2010 Brian Dolbec +# Copyright 2002-2010 Gentoo Technologies, Inc. +# Distributed under the terms of the GNU General Public License v2 or later +# +# $Header$ + +"""'enalyze' is a flexible utility for Gentoo linux which can display various +information about installed packages, such as the USE flags used and the +packages that use them. It can also be used to help rebuild /etc/portage/package.* +files in the event of corruption, and possibly more. +""" + +from __future__ import print_function + +import sys +# This block ensures that ^C interrupts are handled quietly. +try: + import signal + + def exithandler(signum,frame): + signal.signal(signal.SIGINT, signal.SIG_IGN) + signal.signal(signal.SIGTERM, signal.SIG_IGN) + print() + sys.exit(1) + + signal.signal(signal.SIGINT, exithandler) + signal.signal(signal.SIGTERM, exithandler) + signal.signal(signal.SIGPIPE, signal.SIG_DFL) + +except KeyboardInterrupt: + print() + sys.exit(1) + +from gentoolkit import enalyze, errors + +try: + enalyze.main() +except errors.GentoolkitException as err: + if '--debug' in sys.argv: + raise + else: + from gentoolkit import pprinter as pp + sys.stderr.write(pp.error(str(err))) + print() + print("Add '--debug' to global options for traceback.") + sys.exit(1) diff --git a/man/analyse.1 b/man/analyse.1 deleted file mode 100644 index b8db3e8..0000000 --- a/man/analyse.1 +++ /dev/null @@ -1,210 +0,0 @@ -.TH "ANALYSE" "22" "Febuary 2010" "GENTOOLKIT" "" -.SH "NAME" -analyse \- Gentoo Installed Package Analysis Tool - -.SH "SYNOPSIS" -.BI "analyse " "[global\-options] " "module " "[local\-options]" "TARGET" - -.SH "DESCRIPTION" -.B Analyse -Is a collection of modules for analysing the state of installed Gentoo packages for -USE flags or keywords used for installation, and their current masking status. -.br -It can also optionally (re)generate new /etc/portage/package.* files. -.br - -.br -.B CAUTION: -This is beta software and is not yet feature complete. Some features, options or its name -may change in the future. Any files that it generates are saved to your home directory -and will not harm your system without effort on your part. -.br - -.SH "GLOBAL OPTIONS" -.HP -.B \-h, \-\-help -.br -Output a help message. -.HP -.B \-q, \-\-quiet -.br -Be less verbose where possible. In some modules, this option can increase the output speed. -.HP -.B \-C, \-\-no\-color -.br -Do not colorize output. -.HP -.B \-N, \-\-no\-pipe -.br -Turn off automatic pipe detection. Use this option if you do not want -.B analyse -To detect if the output is being directed to the screen or to another program -and adjust color and verbosity accordingly. -.HP -.B \-V, \-\-version -.br -Display \fBGentoolkit\fP's version. Please include this in all bug reports. (see -.B BUGS -below) - -.SH "MODULES" -.B Analyse -Uses a system of modules. Each module has both a long and short name. -The list below uses the notation "\fBmodule (m)\fP", where \fIm\fP is the short name -and \fImodule\fP is the long name. -.P -You can view the -.B help -message for a specific module by using -.BR "\-h" ", " "\-\-help " -as either a global option (after -.B analyse -and before the module name) or as a local option (after the module name). - -.SS -.BI "analyse (a) [OPTIONS] TARGET" -Report on all installed packages for \fITARGET\fP. -.P - -.I R "TARGET" ":" -.HP -.B use -.br -Will analyse the installed with USE flags for output results. -.HP -.B pkguse -.br -Will analyse the USE flags information from the installed pkg's 'PKGUSE' file which contains -only flags settings from /etc/portage/package.use at the time of installation. -.HP -.B keywords -.br -Will analyse the recorded keywords for output results. -.HP -.B unmask -.br -Will analyse the installed packages and portage trees for pkgs that require unmasking and report them. -.br -.P -.I R "LOCAL OPTIONS" ":" -.HP -.B \-u, \-\-unset -.br -Will also include any USE flags used that were not enabled for some packages. -.HP -.B \-v, \-\-verebose -.br -Gives more detail about the results found and the current task being performed. - -.P -.I R "EXAMPLES" ":" -.EX -.HP -analyse a \-\-verbose \-\-unset use -.EE -.br -Report on all use flags used to install the packages. (\-\-unset) Include in the report all flags -that have been used but also were not set enabled for some packages. -(\-\-verbose) Also list the packages that used the USE flag setting. -The report will break down the useage and report the USE flag up to 3 times indicating its -setting {"+","\-"," "= unset} prepended to the flag name. -It will also color the output, red = Disabled, blue = Enabled, plain text = unset -.br - -.SS -.BI "rebuild (r) [OPTIONS] TARGET" -Create a list all packages for \fITARGET\fP settings that are needed for -other than the default settings. - -.I R "TARGET" ":" -.HP -.B use -.br -Will analyse the USE flags for output results. -.HP -.B keywords -.br -Will analyse the keywords for output results. -.HP -.B unmask -.br -Will analyse the installed packages and portage trees for pkgs that require -unmasking and produce output/a new /etc/portage/package.unmask file. -.P -.I R "LOCAL OPTIONS" ":" -.HP -.B \-a, \-\-all -.br -Create files/output for all TARGET(s) found to need it. (not Implemented yet) -.HP -.B \-e, \-\-excact -.br -Will prepend the pkg with = as well as use the version information for the entries. -.br -eg.: =CAT/PKG\-VER flag1 flag2 -.HP -.B \-p, \-\-pretend -.br -Sends the output to the screen instead of a file. -.HP -.B \-v, \-\-verebose -.br -Gives more detail about the results found and the current task being performed. -.P -.I R "EXAMPLES" ":" -.EX -.HP -analyse rebuild \-p use -.EE -.br -Analyse the installed packages database and current system USE flag settings - and output the results in the form of: -.br - -.br -.EX -CAT/PKG \-flag1 \-flag2 flag3 flag4... - -.SS -.BI "clean (c) [OPTIONS] TARGET" -Clean all packages for \fITARGET\fP settings that are found with obsolete settings -for the current settings and pkg ebuild. (not Implemented yet) - -.I R "TARGET" ":" -.HP -.B use -.br -Will analyse the USE flags and /etc/portage/package.use file(s) for entries that -are redundant or no longer used by the pkg. -.HP -.B keywords -.br -Will analyse the keywords and /etc/portage/package.keywords file(s) for entries -that are no longer needed. -.HP -.B unmask -.br -Will analyse the installed packages, /etc/portage/package.unmask file(s) and -portage trees for pkgs that no longer require unmasking. -.P -.I R "LOCAL OPTIONS" ":" -.HP -.B \-a, \-\-all -.br -Clean files/output for all TARGET(s) found to need it. (not Implemented yet) -.HP -.B \-p, \-\-pretend -.br -Sends the output to the screen instead of a file. -.HP -.B \-v, \-\-verebose -.br -Gives more detail about the results found and the current task being performed. - - -.SH "BUGS" -Submit bug reports to http://bugs.gentoo.org. - -.SH "AUTHORS" -.br -Brian Dolbec , 2010 diff --git a/man/enalyze.1 b/man/enalyze.1 new file mode 100644 index 0000000..506d619 --- /dev/null +++ b/man/enalyze.1 @@ -0,0 +1,210 @@ +.TH "ENALYZE" "22" "Febuary 2010" "GENTOOLKIT" "" +.SH "NAME" +enalyze \- Gentoo Installed Package Analysis Tool + +.SH "SYNOPSIS" +.BI "enalyze " "[global\-options] " "module " "[local\-options]" "TARGET" + +.SH "DESCRIPTION" +.B Enalyze +Is a collection of modules for analyzing the state of installed Gentoo packages for +USE flags or keywords used for installation, and their current masking status. +.br +It can also optionally (re)generate new /etc/portage/package.* files. +.br + +.br +.B CAUTION: +This is beta software and is not yet feature complete. Some features, options +may change in the future. Any files that it generates are saved to your home directory +and will not harm your system without effort on your part. +.br + +.SH "GLOBAL OPTIONS" +.HP +.B \-h, \-\-help +.br +Output a help message. +.HP +.B \-q, \-\-quiet +.br +Be less verbose where possible. In some modules, this option can increase the output speed. +.HP +.B \-C, \-\-no\-color +.br +Do not colorize output. +.HP +.B \-N, \-\-no\-pipe +.br +Turn off automatic pipe detection. Use this option if you do not want +.B enalyze +To detect if the output is being directed to the screen or to another program +and adjust color and verbosity accordingly. +.HP +.B \-V, \-\-version +.br +Display \fBGentoolkit\fP's version. Please include this in all bug reports. (see +.B BUGS +below) + +.SH "MODULES" +.B Enalyze +Uses a system of modules. Each module has both a long and short name. +The list below uses the notation "\fBmodule (m)\fP", where \fIm\fP is the short name +and \fImodule\fP is the long name. +.P +You can view the +.B help +message for a specific module by using +.BR "\-h" ", " "\-\-help " +as either a global option (after +.B enalyze +and before the module name) or as a local option (after the module name). + +.SS +.BI "analyze (a) [OPTIONS] TARGET" +Report on all installed packages for \fITARGET\fP. +.P + +.I R "TARGET" ":" +.HP +.B use +.br +Will analyze the installed with USE flags for output results. +.HP +.B pkguse +.br +Will analyze the USE flags information from the installed pkg's 'PKGUSE' file which contains +only flags settings from /etc/portage/package.use at the time of installation. +.HP +.B keywords +.br +Will analyze the recorded keywords for output results. +.HP +.B unmask +.br +Will analyze the installed packages and portage trees for pkgs that require unmasking and report them. +.br +.P +.I R "LOCAL OPTIONS" ":" +.HP +.B \-u, \-\-unset +.br +Will also include any USE flags used that were not enabled for some packages. +.HP +.B \-v, \-\-verebose +.br +Gives more detail about the results found and the current task being performed. + +.P +.I R "EXAMPLES" ":" +.EX +.HP +enalyze a \-\-verbose \-\-unset use +.EE +.br +Report on all use flags used to install the packages. (\-\-unset) Include in the report all flags +that have been used but also were not set enabled for some packages. +(\-\-verbose) Also list the packages that used the USE flag setting. +The report will break down the useage and report the USE flag up to 3 times indicating its +setting {"+","\-"," "= unset} prepended to the flag name. +It will also color the output, red = Disabled, blue = Enabled, plain text = unset +.br + +.SS +.BI "rebuild (r) [OPTIONS] TARGET" +Create a list all packages for \fITARGET\fP settings that are needed for +other than the default settings. + +.I R "TARGET" ":" +.HP +.B use +.br +Will analyze the USE flags for output results. +.HP +.B keywords +.br +Will analyze the keywords for output results. +.HP +.B unmask +.br +Will analyze the installed packages and portage trees for pkgs that require +unmasking and produce output/a new /etc/portage/package.unmask file. +.P +.I R "LOCAL OPTIONS" ":" +.HP +.B \-a, \-\-all +.br +Create files/output for all TARGET(s) found to need it. (not Implemented yet) +.HP +.B \-e, \-\-excact +.br +Will prepend the pkg with = as well as use the version information for the entries. +.br +eg.: =CAT/PKG\-VER flag1 flag2 +.HP +.B \-p, \-\-pretend +.br +Sends the output to the screen instead of a file. +.HP +.B \-v, \-\-verebose +.br +Gives more detail about the results found and the current task being performed. +.P +.I R "EXAMPLES" ":" +.EX +.HP +enalyze rebuild \-p use +.EE +.br +Analyze the installed packages database and current system USE flag settings + and output the results in the form of: +.br + +.br +.EX +CAT/PKG \-flag1 \-flag2 flag3 flag4... + +.SS +.BI "clean (c) [OPTIONS] TARGET" +Clean all packages for \fITARGET\fP settings that are found with obsolete settings +for the current settings and pkg ebuild. (not Implemented yet) + +.I R "TARGET" ":" +.HP +.B use +.br +Will analyze the USE flags and /etc/portage/package.use file(s) for entries that +are redundant or no longer used by the pkg. +.HP +.B keywords +.br +Will analyze the keywords and /etc/portage/package.keywords file(s) for entries +that are no longer needed. +.HP +.B unmask +.br +Will analyze the installed packages, /etc/portage/package.unmask file(s) and +portage trees for pkgs that no longer require unmasking. +.P +.I R "LOCAL OPTIONS" ":" +.HP +.B \-a, \-\-all +.br +Clean files/output for all TARGET(s) found to need it. (not Implemented yet) +.HP +.B \-p, \-\-pretend +.br +Sends the output to the screen instead of a file. +.HP +.B \-v, \-\-verebose +.br +Gives more detail about the results found and the current task being performed. + + +.SH "BUGS" +Submit bug reports to http://bugs.gentoo.org. + +.SH "AUTHORS" +.br +Brian Dolbec , 2010 diff --git a/pym/gentoolkit/analyse/__init__.py b/pym/gentoolkit/analyse/__init__.py deleted file mode 100644 index 46d6185..0000000 --- a/pym/gentoolkit/analyse/__init__.py +++ /dev/null @@ -1,128 +0,0 @@ -#!/usr/bin/python -# -# Copyright 2010 Brian Dolbec -# Copyright(c) 2010, Gentoo Foundation -# Copyright 2003-2004 Karl Trygve Kalleberg -# Licensed under the GNU General Public License, v2 -# -# $Header: $ - -"""Gentoo's installed packages analysis and repair tool""" - - -# Move to Imports section after Python 2.6 is stable - - -__docformat__ = 'epytext' -# version is dynamically set by distutils sdist -__version__ = "svn" -__productname__ = "analyse" -__authors__ = ( - 'Brian Dolbec, ' - -) - -# make an exportable copy of the info for help output -MODULE_INFO = { - "__docformat__": __docformat__, - "__doc__": __doc__, - "__version__": __version__, - "__productname__": __productname__, - "__authors__": __authors__ - -} - -import errno -import sys -import time -from getopt import getopt, GetoptError - -import portage - -import gentoolkit as gen -from gentoolkit import errors -from gentoolkit import pprinter as pp -from gentoolkit.base import (initialize_configuration, split_arguments, - parse_global_options, print_help) -from gentoolkit.formatters import format_options - - -NAME_MAP = { - 'a': 'analyse', - 'r': 'rebuild' -} - -FORMATTED_OPTIONS = ( - (" (a)nalyse", - "analyses the installed PKG database USE flag or keyword useage"), - (" (r)ebuild", - "analyses the Installed PKG database and generates files suitable"), - (" ", - "to replace corrupted or missing /etc/portage/package.* files") - ) - -def expand_module_name(module_name): - """Returns one of the values of NAME_MAP or raises KeyError""" - - if module_name == 'list': - # list is a Python builtin type, so we must rename our module - return 'list_' - elif module_name in NAME_MAP.values(): - return module_name - else: - return NAME_MAP[module_name] - - -def main(): - """Parse input and run the program.""" - - short_opts = "hqCNV" - long_opts = ( - 'help', 'quiet', 'nocolor', 'no-color', 'no-pipe', 'version', 'debug' - ) - - initialize_configuration() - - try: - global_opts, args = getopt(sys.argv[1:], short_opts, long_opts) - except GetoptError as err: - sys.stderr.write(" \n") - sys.stderr.write(pp.error("Global %s\n" % err)) - print_help(MODULE_INFO, FORMATTED_OPTIONS, with_description=False) - sys.exit(2) - - # Parse global options - need_help = parse_global_options(global_opts, args, MODULE_INFO, FORMATTED_OPTIONS) - - if gen.CONFIG['quiet']: - gen.CONFIG['verbose'] = False - - try: - module_name, module_args = split_arguments(args) - except IndexError: - print_help(MODULE_INFO, FORMATTED_OPTIONS) - sys.exit(2) - - if need_help: - module_args.append('--help') - - try: - expanded_module_name = expand_module_name(module_name) - except KeyError: - sys.stderr.write(pp.error("Unknown module '%s'" % module_name)) - print_help(MODULE_INFO, FORMATTED_OPTIONS, with_description=False) - sys.exit(2) - - try: - loaded_module = __import__( - expanded_module_name, globals(), locals(), [], -1 - ) - loaded_module.main(module_args) - except portage.exception.AmbiguousPackageName as err: - raise errors.GentoolkitAmbiguousPackage(err.args[0]) - except IOError as err: - if err.errno != errno.EPIPE: - raise - -if __name__ == '__main__': - main() diff --git a/pym/gentoolkit/analyse/analyse.py b/pym/gentoolkit/analyse/analyse.py deleted file mode 100644 index e2628e3..0000000 --- a/pym/gentoolkit/analyse/analyse.py +++ /dev/null @@ -1,448 +0,0 @@ -#!/usr/bin/python -# -# Copyright 2010 Brian Dolbec -# Copyright(c) 2010, Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 -# - -"""Provides a breakdown list of USE flags or keywords used and by -what packages according to the Installed package database""" - -from __future__ import print_function - -import sys - -import gentoolkit -from gentoolkit.dbapi import PORTDB, VARDB -from gentoolkit.analyse.base import ModuleBase -from gentoolkit import pprinter as pp -from gentoolkit.flag import get_installed_use, get_flags -from gentoolkit.analyse.lib import FlagAnalyzer, KeywordAnalyser -from gentoolkit.analyse.output import nl, AnalysisPrinter -from gentoolkit.package import Package -from gentoolkit.helpers import get_installed_cpvs - -import portage - - -def gather_flags_info( - cpvs=None, - system_flags=None, - include_unset=False, - target="USE", - use_portage=False, - # override-able for testing - _get_flags=get_flags, - _get_used=get_installed_use - ): - """Analyse the installed pkgs USE flags for frequency of use - - @type cpvs: list - @param cpvs: optional list of [cat/pkg-ver,...] to analyse or - defaults to entire installed pkg db - @type: system_flags: list - @param system_flags: the current default USE flags as defined - by portage.settings["USE"].split() - @type include_unset: bool - @param include_unset: controls the inclusion of unset USE flags in the report. - @type target: string - @param target: the environment variable being analysed - one of ["USE", "PKGUSE"] - @type _get_flags: function - @param _get_flags: ovride-able for testing, - defaults to gentoolkit.analyse.lib.get_flags - @param _get_used: ovride-able for testing, - defaults to gentoolkit.analyse.lib.get_installed_use - @rtype dict. {flag:{"+":[cat/pkg-ver,...], "-":[cat/pkg-ver,...], "unset":[]} - """ - if cpvs is None: - cpvs = VARDB.cpv_all() - # pass them in to override for tests - flags = FlagAnalyzer(system_flags, - filter_defaults=False, - target=target, - _get_flags=_get_flags, - _get_used=get_installed_use - ) - flag_users = {} - for cpv in cpvs: - if cpv.startswith("virtual"): - continue - if use_portage: - plus, minus, unset = flags.analyse_cpv(cpv) - else: - pkg = Package(cpv) - plus, minus, unset = flags.analyse_pkg(pkg) - for flag in plus: - if flag in flag_users: - flag_users[flag]["+"].append(cpv) - else: - flag_users[flag] = {"+": [cpv], "-": []} - for flag in minus: - if flag in flag_users: - flag_users[flag]["-"].append(cpv) - else: - flag_users[flag] = {"+":[], "-": [cpv]} - if include_unset: - for flag in unset: - if flag in flag_users: - if "unset" in flag_users[flag]: - flag_users[flag]["unset"].append(cpv) - else: - flag_users[flag]["unset"] = [cpv] - else: - flag_users[flag] = {"+": [], "-": [], "unset": [cpv]} - return flag_users - - -def gather_keywords_info( - cpvs=None, - system_keywords=None, - use_portage=False, - # override-able for testing - keywords=portage.settings["ACCEPT_KEYWORDS"], - analyser = None - ): - """Analyse the installed pkgs 'keywords' for frequency of use - - @param cpvs: optional list of [cat/pkg-ver,...] to analyse or - defaults to entire installed pkg db - @param system_keywords: list of the system keywords - @param keywords: user defined list of keywords to check and report on - or reports on all relevant keywords found to have been used. - @param _get_kwds: overridable function for testing - @param _get_used: overridable function for testing - @rtype dict. {keyword:{"stable":[cat/pkg-ver,...], "testing":[cat/pkg-ver,...]} - """ - if cpvs is None: - cpvs = VARDB.cpv_all() - keyword_users = {} - for cpv in cpvs: - if cpv.startswith("virtual"): - continue - if use_portage: - keyword = analyser.get_inst_keyword_cpv(cpv) - else: - pkg = Package(cpv) - keyword = analyser.get_inst_keyword_pkg(pkg) - #print "returned keyword =", cpv, keyword, keyword[0] - key = keyword[0] - if key in ["~", "-"]: - _kwd = keyword[1:] - if _kwd in keyword_users: - if key in ["~"]: - keyword_users[_kwd]["testing"].append(cpv) - elif key in ["-"]: - #print "adding cpv to missing:", cpv - keyword_users[_kwd]["missing"].append(cpv) - else: - if key in ["~"]: - keyword_users[_kwd] = {"stable": [], - "testing": [cpv], "missing": []} - elif key in ["-"]: - keyword_users[_kwd] = {"stable": [], - "testing": [], "missing": [cpv]} - else: - keyword_users[_kwd] = {"stable": [cpv], - "testing": [], "missing": []} - elif keyword in keyword_users: - keyword_users[keyword]["stable"].append(cpv) - else: - keyword_users[keyword] = { - "stable": [cpv], - "testing": [], - "missing": [] - } - return keyword_users - - -class Analyse(ModuleBase): - """Installed db analysis tool to query the installed databse - and produce/output stats for USE flags or keywords/mask. - The 'rebuild' action output is in the form suitable for file type output - to create a new package.use, package.keywords, package.unmask - type files in the event of needing to rebuild the - /etc/portage/* user configs - """ - def __init__(self): - ModuleBase.__init__(self) - self.module_name = "analyse" - self.options = { - "flags": False, - "keywords": False, - "packages": False, - "unset": False, - "verbose": False, - "quiet": False, - 'prefix': False, - 'portage': True - } - self.module_opts = { - "-f": ("flags", "boolean", True), - "--flags": ("flags", "boolean", True), - "-k": ("keywords", "boolean", True), - "--keywords": ("keywords", "boolean", True), - "-u": ("unset", "boolean", True), - "--unset": ("unset", "boolean", True), - "-v": ("verbose", "boolean", True), - "--verbose": ("verbose", "boolean", True), - "-p": ("prefix", "boolean", True), - "--prefix": ("prefix", "boolean", True), - "-G": ("portage", "boolean", False), - "--portage": ("portage", "boolean", False), - } - self.formatted_options = [ - (" -h, --help", "Outputs this useage message"), - (" -a, --analyse", - "Action, sets the module to gather data and output the"), - ("", "formatted stats/information to the screen"), - (" -u, --unset", - "Additionally include any unset USE flags and the packages"), - ("", "that could use them"), - (" -v, --verbose", - "Used in the analyse action to output more detailed information"), - (" -p, --prefix", - "Used for testing purposes only, runs report using " + - "a prefix keyword and 'prefix' USE flag"), - #(" -G, --portage", - #"Use portage directly instead of gentoolkit's Package " + - #"object for some operations. Usually a little faster."), - ] - self.formatted_args = [ - (" use", - "Causes the action to analyse the installed packages USE flags"), - (" pkguse", - "Causes the action to analyse the installed packages PKGUSE flags"), - (" ", - "These are flags that have been set in /etc/portage/package.use"), - (" keywords", - "Causes the action to analyse the installed packages keywords"), - (" packages", - "Causes the action to analyse the installed packages and the"), - (" ", - "USE flags they were installed with"), - ] - self.short_opts = "huvpG" - self.long_opts = ("help", "unset", "verbose", "prefix") #, "portage") - self.need_queries = True - self.arg_spec = "Target" - self.arg_options = ['use', 'pkguse','keywords', 'packages'] - self.arg_option = False - self.warning = ( - " CAUTION", - "This is beta software and some features/options are incomplete,", - "some features may change in future releases includig its name.", - "Feedback will be appreciated, http://bugs.gentoo.org") - - - def run(self, input_args, quiet=False): - """runs the module - - @param input_args: input arguments to be parsed - """ - query = self.main_setup(input_args) - query = self.validate_query(query) - self.set_quiet(quiet) - if query in ["use", "pkguse"]: - self.analyse_flags(query) - elif query in ["keywords"]: - self.analyse_keywords() - elif query in ["packages"]: - self.analyse_packages() - - def analyse_flags(self, target): - """This will scan the installed packages db and analyse the - USE flags used for installation and produce a report on how - they were used. - - @type target: string - @param target: the target to be analysed, one of ["use", "pkguse"] - """ - system_use = portage.settings["USE"].split() - self.printer = AnalysisPrinter( - "use", - self.options["verbose"], - system_use) - if self.options["verbose"]: - cpvs = VARDB.cpv_all() - #cpvs = get_installed_cpvs() - #print "Total number of installed ebuilds =", len(cpvs) - flag_users = gather_flags_info(cpvs, system_use, - self.options["unset"], target=target.upper(), - use_portage=self.options['portage']) - else: - cpvs = get_installed_cpvs() - flag_users = gather_flags_info(cpvs, system_flags=system_use, - include_unset=self.options["unset"], target=target.upper(), - use_portage=self.options['portage']) - #print flag_users - flag_keys = sorted(flag_users) - if self.options["verbose"]: - print(" Flag System #pkgs cat/pkg-ver") - blankline = nl - elif not self.options['quiet']: - print(" Flag System #pkgs") - blankline = lambda: None - for flag in flag_keys: - flag_pos = flag_users[flag]["+"] - if len(flag_pos): - self.printer(flag, "+", flag_pos) - #blankline() - flag_neg = flag_users[flag]["-"] - if len(flag_neg): - self.printer(flag, "-", flag_neg) - #blankline() - if "unset" in flag_users[flag] and flag_users[flag]["unset"]: - flag_unset = flag_users[flag]["unset"] - self.printer(flag, "unset", flag_unset) - #blankline() - if not self.options['quiet']: - print("===================================================") - print("Total number of flags in report =", - pp.output.red(str(len(flag_keys)))) - if self.options["verbose"]: - print("Total number of installed ebuilds =", - pp.output.red(str(len([x for x in cpvs])))) - print() - - - def analyse_keywords(self, keywords=None): - """This will scan the installed packages db and analyse the - keywords used for installation and produce a report on them. - """ - print() - system_keywords = portage.settings["ACCEPT_KEYWORDS"] - arch = portage.settings["ARCH"] - if self.options["prefix"]: - # build a new keyword for testing - system_keywords = "~" + arch + "-linux" - if self.options["verbose"] or self.options["prefix"]: - print("Current system ARCH =", arch) - print("Current system ACCEPT_KEYWORDS =", system_keywords) - system_keywords = system_keywords.split() - self.printer = AnalysisPrinter( - "keywords", - self.options["verbose"], - system_keywords) - self.analyser = KeywordAnalyser( arch, system_keywords, VARDB) - #self.analyser.set_order(portage.settings["USE"].split()) - # only for testing - test_use = portage.settings["USE"].split() - if self.options['prefix'] and 'prefix' not in test_use: - print("ANALYSE_KEYWORDS() 'prefix' flag not found in system", - "USE flags!!! appending for testing") - print() - test_use.append('prefix') - self.analyser.set_order(test_use) - # /end testing - - if self.options["verbose"]: - cpvs = VARDB.cpv_all() - #print "Total number of installed ebuilds =", len(cpvs) - keyword_users = gather_keywords_info( - cpvs=cpvs, - system_keywords=system_keywords, - use_portage=self.options['portage'], - keywords=keywords, analyser = self.analyser - ) - blankline = nl - else: - keyword_users = gather_keywords_info( - system_keywords=system_keywords, - use_portage=self.options['portage'], - keywords=keywords, - analyser = self.analyser - ) - blankline = lambda: None - #print keyword_users - keyword_keys = sorted(keyword_users) - if self.options["verbose"]: - print(" Keyword System #pkgs cat/pkg-ver") - elif not self.options['quiet']: - print(" Keyword System #pkgs") - for keyword in keyword_keys: - kwd_stable = keyword_users[keyword]["stable"] - if len(kwd_stable): - self.printer(keyword, " ", kwd_stable) - blankline() - kwd_testing = keyword_users[keyword]["testing"] - if len(kwd_testing): - self.printer(keyword, "~", kwd_testing) - blankline() - kwd_missing = keyword_users[keyword]["missing"] - if len(kwd_missing): - self.printer(keyword, "-", kwd_missing) - blankline - if not self.options['quiet']: - if self.analyser.mismatched: - print("_________________________________________________") - print(("The following packages were found to have a \n" + - "different recorded ARCH than the current system ARCH")) - for cpv in self.analyser.mismatched: - print("\t", pp.cpv(cpv)) - print("===================================================") - print("Total number of keywords in report =", - pp.output.red(str(len(keyword_keys)))) - if self.options["verbose"]: - print("Total number of installed ebuilds =", - pp.output.red(str(len(cpvs)))) - print() - - - def analyse_packages(self): - """This will scan the installed packages db and analyse the - USE flags used for installation and produce a report. - - @type target: string - @param target: the target to be analysed, one of ["use", "pkguse"] - """ - system_use = portage.settings["USE"].split() - if self.options["verbose"]: - cpvs = VARDB.cpv_all() - key_width = 45 - else: - cpvs = get_installed_cpvs() - key_width = 1 - - self.printer = AnalysisPrinter( - "packages", - self.options["verbose"], - key_width=key_width) - - cpvs = sorted(cpvs) - flags = FlagAnalyzer( - system=system_use, - filter_defaults=False, - target="USE" - ) - - if self.options["verbose"]: - print(" cat/pkg-ver USE Flags") - # "app-emulation/emul-linux-x86-sdl-20100915 ...." - blankline = nl - elif not self.options['quiet']: - print(" cat/pkg-ver USE Flags") - blankline = lambda: None - for cpv in cpvs: - (flag_plus, flag_neg, unset) = flags.analyse_cpv(cpv) - if self.options["unset"]: - self.printer(cpv, "", (flag_plus, flag_neg, unset)) - else: - self.printer(cpv, "", (flag_plus, flag_neg, [])) - if not self.options['quiet']: - print("===================================================") - print("Total number of installed ebuilds =", - pp.output.red(str(len([x for x in cpvs])))) - print() - - -def main(input_args): - """Common starting method by the analyse master - unless all modules are converted to this class method. - - @param input_args: input args as supplied by equery master module. - """ - query_module = Analyse() - query_module.run(input_args, gentoolkit.CONFIG['quiet']) - -# vim: set ts=4 sw=4 tw=79: diff --git a/pym/gentoolkit/analyse/base.py b/pym/gentoolkit/analyse/base.py deleted file mode 100644 index a3f3fed..0000000 --- a/pym/gentoolkit/analyse/base.py +++ /dev/null @@ -1,148 +0,0 @@ -# Copyright(c) 2009, Gentoo Foundation -# -# Copyright 2010 Brian Dolbec -# Copyright(c) 2010, Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 -# -# $Header: $ - -"""Analyse Base Module class to hold common module operation functions -""" - -from __future__ import print_function - -__docformat__ = 'epytext' - - -import errno -import sys -import time -from getopt import gnu_getopt, GetoptError - -import gentoolkit.pprinter as pp -from gentoolkit.formatters import format_options -from gentoolkit.base import mod_usage -from gentoolkit import CONFIG - -class ModuleBase(object): - """Analyse base module class to parse module options print module help, etc..""" - - def __init__(self): - self.module_name = None - self.options = {} - self.formatted_options = None - self.short_opts = None - self.long_opts = None - self.module_opts = {} - self.warning = None - self.need_queries = True - self.saved_verbose = None - - - def print_help(self, with_description=True): - """Print description, usage and a detailed help message. - - @type with_description: bool - @param with_description: if true, print module's __doc__ string - """ - - if with_description: - print() - print(__doc__.strip()) - print() - if self.warning: - print() - for line in self.warning: - sys.stderr.write(pp.warn(line)) - print() - print(mod_usage(mod_name=self.module_name, arg=self.arg_spec, optional=self.arg_option)) - print() - print(pp.command("options")) - print(format_options( self.formatted_options )) - if self.formatted_args: - print() - print(pp.command(self.arg_spec)) - print(format_options(self.formatted_args)) - print() - - def parse_module_options(self, module_opts): - """Parse module options and update self.options""" - - opts = (x[0] for x in module_opts) - posargs = (x[1] for x in module_opts) - for opt, posarg in zip(opts, posargs): - if opt in ('-h', '--help'): - self.print_help() - sys.exit(0) - opt_name, opt_type, opt_setting = self.module_opts[opt] - if opt_type == 'boolean': - self.options[opt_name] = opt_setting - elif opt_type == 'int': - if posarg.isdigit(): - val = int(posarg) - else: - print() - err = "Module option %s requires integer (got '%s')" - sys.stdout.write(pp.error(err % (opt,posarg))) - print() - self.print_help(with_description=False) - sys.exit(2) - self.options[opt_name] = val - - def set_quiet(self, quiet): - """sets the class option["quiet"] and option["verbose"] accordingly""" - if quiet == self.options['quiet']: - return - if self.saved_verbose: - # detected a switch - verbose = self.options['verbose'] - self.options['verbose'] = self.saved_verbose - self.saved_verbose = verbose - elif quiet: - self.saved_verbose = self.options['verbose'] - self.options['verbose'] = False - self.options['quiet'] = quiet - return - - def validate_query(self, query, depth=0): - """check that the query meets the modules TargetSpec - If not it attempts to reduce it to a valid TargetSpec - or prints the help message and exits - """ - if depth > 1: - return [] - if len(query) > 1: - query = list(set(self.arg_options).intersection(query)) - #print "reduced query =", query - query = self.validate_query(query, depth+1) - if isinstance(query, list): - query = query[0] - if query not in self.arg_options: - print() - print(pp.error( - "Error starting module. Incorrect or No TargetSpec specified!" - )) - print("query = ", query) - self.print_help() - sys.exit(2) - return query - - - def main_setup(self, input_args): - """Parse input and prepares the program""" - - try: - module_opts, queries = gnu_getopt(input_args, self.short_opts, self.long_opts) - except GetoptError as err: - sys.stderr.write(pp.error("Module %s" % err)) - print() - self.print_help(with_description=False) - sys.exit(2) - self.parse_module_options(module_opts) - if self.need_queries and not queries: - self.print_help() - sys.exit(2) - return queries - - -# vim: set ts=4 sw=4 tw=79: diff --git a/pym/gentoolkit/analyse/lib.py b/pym/gentoolkit/analyse/lib.py deleted file mode 100644 index 901d757..0000000 --- a/pym/gentoolkit/analyse/lib.py +++ /dev/null @@ -1,354 +0,0 @@ -#!/usr/bin/python -# -# Copyright 2010 Brian Dolbec -# Copyright(c) 2010, Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 -# - - -"""Provides support functions to analyse modules""" - -import sys - -from gentoolkit.dbapi import PORTDB, VARDB -from gentoolkit import errors -from gentoolkit.keyword import reduce_keywords -from gentoolkit.flag import (reduce_flags, get_flags, get_all_cpv_use, - filter_flags, get_installed_use, get_iuse) -#from gentoolkit.package import Package - -import portage - - -class FlagAnalyzer(object): - """Specialty functions for analysing an installed package's - USE flags. Can be used for single or mulitple use without - needing to be reset unless the system USE flags are changed. - - @type system: list or set - @param system: the default system USE flags. - @type _get_flags: function - @param _get_flags: Normally defaulted, can be overriden for testing - @type _get_used: function - @param _get_used: Normally defaulted, can be overriden for testing - """ - def __init__(self, - system, - filter_defaults=False, - target="USE", - _get_flags=get_flags, - _get_used=get_installed_use - ): - self.get_flags = _get_flags - self.get_used = _get_used - self.filter_defaults = filter_defaults - self.target = target - self.reset(system) - - def reset(self, system): - """Resets the internal system USE flags and use_expand variables - to the new setting. The use_expand variable is handled internally. - - @type system: list or set - @param system: the default system USE flags. - """ - self.system = set(system) - self.use_expand = portage.settings['USE_EXPAND'].lower().split() - - def analyse_cpv(self, cpv): - """Gets all relavent USE flag info for a cpv and breaks them down - into 3 sets, plus (package.use enabled), minus ( package.use disabled), - unset. - - @param cpv: string. 'cat/pkg-ver' - @rtype tuple of sets - @return (plus, minus, unset) sets of USE flags - """ - installed = set(self.get_used(cpv, self.target)) - iuse = set(reduce_flags(self.get_flags(cpv))) - return self._analyse(installed, iuse) - - def _analyse(self, installed, iuse): - """Analyses the supplied info and returns the flag settings - that differ from the defaults - - @type installed: set - @param installed: the installed with use flags - @type iuse: set - @param iuse: the current ebuilds IUSE - """ - defaults = self.system.intersection(iuse) - usedflags = iuse.intersection(set(installed)) - if self.filter_defaults: - plus = usedflags.difference(defaults) - else: - plus = usedflags - minus = defaults.difference(usedflags) - unset = iuse.difference(defaults, plus, minus) - cleaned_unset = self.remove_expanding(unset) - return (plus, minus, cleaned_unset) - - def analyse_pkg(self, pkg): - """Gets all relevent USE flag info for a pkg and breaks them down - into 3 sets, plus (package.use enabled), minus ( package.use disabled), - unset. - - @param pkg: gentoolkit.package.Package object - @rtype tuple of sets - @return (plus, minus, unset) sets of USE flags - """ - installed = set(self.pkg_used(pkg)) - print("installed =", installed) - iuse = set(reduce_flags(self.pkg_flags(pkg))) - print("iuse =", iuse) - return self._analyse(installed, iuse) - - def pkg_used(self, pkg): - if self.target == "USE": - return pkg.use().split() - return pkg.environment(self.target).split() - - def pkg_flags(self, pkg): - final_use, use_expand_hidden, usemasked, useforced = \ - get_all_cpv_use(pkg.cpv) - flags = pkg.environment("IUSE", prefer_vdb=False).split() - return filter_flags(flags, use_expand_hidden, usemasked, useforced) - - def redundant(self, cpv, iuse): - """Checks for redundant settings. - future function. Not yet implemented. - """ - pass - - def remove_expanding(self, flags): - """Remove unwanted USE_EXPAND flags - from unset IUSE sets - - @param flags: short list or set of USE flags - @rtype set - @return USE flags - """ - _flags = set(flags) - for expander in self.use_expand: - for flag in flags: - if expander in flag: - _flags.remove(flag) - if not _flags: - break - return _flags - - -class KeywordAnalyser(object): - """Specialty functions for analysing the installed package db for - keyword useage and the packages that used them. - - Note: should be initialized with the internal set_order() before use. - See internal set_order() for more details. - This class of functions can be used for single cpv checks or - used repeatedly for an entire package db. - - @type arch: string - @param arch: the system ARCH setting - @type accept_keywords: list - @param accept_keywords: eg. ['x86', '~x86'] - @type get_aux: function, defaults to: VARDB.aux_get - @param vardb: vardb class of functions, needed=aux_get() - to return => KEYWORDS & USE flags for a cpv - = aux_get(cpv, ["KEYWORDS", "USE"]) - """ - - # parsing order to determine appropriate keyword used for installation - normal_order = ['stable', 'testing', 'prefix', 'testing_prefix', 'missing'] - prefix_order = ['prefix', 'testing_prefix', 'stable', 'testing', 'missing'] - parse_range = list(range(len(normal_order))) - - - def __init__(self, arch, accept_keywords, vardb=VARDB): - self.arch = arch - self.accept_keywords = accept_keywords - self.vardb = vardb - self.prefix = '' - self.parse_order = None - self.check_key = { - 'stable': self._stable, - 'testing': self._testing, - 'prefix': self._prefix, - 'testing_prefix': self._testing_prefix, - 'missing': self._missing - } - self.mismatched = [] - - def determine_keyword(self, keywords, used, cpv): - """Determine the keyword from the installed USE flags and - the KEYWORDS that was used to install a package. - - @param keywords: list of keywords available to install a pkg - @param used: list of USE flalgs recorded for the installed pkg - @rtype: string - @return a keyword or null string - """ - used = set(used) - kwd = None - result = '' - if keywords: - absolute_kwds = reduce_keywords(keywords) - kwd = list(used.intersection(absolute_kwds)) - #if keywords == ['~ppc64']: - #print "Checked keywords for kwd", keywords, used, "kwd =", kwd - if not kwd: - #print "Checking for kwd against portage.archlist" - absolute_kwds = reduce_keywords(keywords) - # check for one against archlist then re-check - kwd = list(absolute_kwds.intersection(portage.archlist)) - #print "determined keyword =", kwd - if len(kwd) == 1: - key = kwd[0] - #print "determined keyword =", key - elif not kwd: - #print "kwd != 1", kwd, cpv - result = self._missing(self.keyword, keywords) - else: # too many, try to narrow them dowm - #print "too many kwd's, trying to match against arch" - _kwd = list(set(kwd).intersection(self.arch)) - key = '' - if _kwd: - #print "found one! :)", _kwd - key = _kwd - else: # try re-running the short list against archlist - #print "Checking kwd for _kwd against portage.archlist" - _kwd = list(set(kwd).intersection(portage.archlist)) - if _kwd and len(_kwd) == 1: - #print "found one! :)", _kwd - key = _kwd[0] - else: - #print " :( didn't work, _kwd =", _kwd, "giving up on:", cpv - result = self._missing(self.keyword, keywords) - i = 0 - while not result and i in self.parse_range: - parsekey = self.parse_order[i] - result = self.check_key[parsekey](key, keywords) - i += 1 - return result - - def _stable(self, key, keywords): - """test for a normal stable keyword""" - if key in keywords: - return key - return '' - - def _testing(self, key, keywords): - """test for a normal testing keyword""" - if ("~" + key) in keywords: - return "~" + key - return '' - - def _prefix(self, key, keywords): - """test for a stable prefix keyword""" - if not self.prefix: - return '' - _key = '-'.join([key, self.prefix]) - if _key in keywords: - #print key, "is in", keywords - return _key - return '' - - def _testing_prefix(self, key, keywords): - """test for a testing prefix keyword""" - if not self.prefix: - return '' - _key = "~" +'-'.join([key, self.prefix]) - if _key in keywords: - #print key, "is in", keywords - return _key - return '' - - def _missing(self, key, keywords): - """generates a missing keyword to return""" - if self.prefix and key != self.keyword: - _key = '-'.join([key, self.prefix]) - else: - _key = '-' + key - #print "_missisng :( _key =", _key - return _key - - def get_inst_keyword_cpv(self, cpv): - """Determines the installed with keyword for cpv - - @type cpv: string - @param cpv: an installed CAT/PKG-VER - @rtype: string - @returns a keyword determined to have been used to install cpv - """ - keywords, used = self.vardb.aux_get(cpv, ["KEYWORDS", "USE"]) - keywords = keywords.split() - used = used.split() - return self._parse(keywords, used, cpv=cpv) - - def get_inst_keyword_pkg(self, pkg): - """Determines the installed with keyword for cpv - - @param pkg: gentoolkit.package.Package object - @rtype: string - @returns a keyword determined to have been used to install cpv - """ - keywords, used = pkg.environment(["KEYWORDS", "USE"], - prefer_vdb=True, fallback=False) - keywords = keywords.split() - used = used.split() - return self._parse(keywords, used, pkg=pkg) - - def _parse(self, keywords, used, pkg=None, cpv=None): - if pkg: - _cpv = pkg.cpv - else: - _cpv = cpv - if not self.parse_order: - self.set_order(used) - keyword = self.keyword - # sanity check - if self.arch not in used: - #print "Found a mismatch = ", cpv, self.arch, used - self.mismatched.append(_cpv) - if keyword in keywords: - #print "keyword", keyword, "is in", keywords - return keyword - elif "~"+keyword in keywords: - #print "~keyword", keyword, "is in", keywords - return "~"+keyword - else: - keyword = self.determine_keyword(keywords, used, _cpv) - if not keyword: - raise errors.GentoolkitUnknownKeyword(_cpv, ' '.join(keywords), used) - return keyword - - def set_order(self, used): - """Used to set the parsing order to determine a keyword - used for installation. - - This is needed due to the way prefix arch's and keywords - work with portage. It looks for the 'prefix' flag. A positive result - sets it to the prefix order and keyword. - - @type used: list - @param used: a list of pkg USE flags or the system USE flags""" - if 'prefix' in used: - #print "SET_ORDER() Setting parse order to prefix" - prefix = None - self.parse_order = self.prefix_order - for key in self.accept_keywords: - #print "SET_ORDER() '"+key+"'" - if '-' in key: - #print "SET_ORDER()found prefix keyword :", key - if self.arch in key: - prefix = key.split('-')[1] - #print "prefix =", prefix - self.prefix = prefix - self.keyword = '-'.join([self.arch, prefix]) - else: - #print "SET_ORDER() Setting parse order to normal" - self.parse_order = self.normal_order - self.keyword = self.arch - #print "SET_ORDER() completed: prefix =", self.prefix, ", keyword =", \ - # self.keyword, "parse order =",self.parse_order - #print - diff --git a/pym/gentoolkit/analyse/output.py b/pym/gentoolkit/analyse/output.py deleted file mode 100644 index b193d54..0000000 --- a/pym/gentoolkit/analyse/output.py +++ /dev/null @@ -1,297 +0,0 @@ -#!/usr/bin/python -# -# Copyright(c) 2010, Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 -# - -"""Provides various output classes and functions for -both screen and file output -""" - -from __future__ import print_function - -import time - -import gentoolkit -from gentoolkit import pprinter as pp -from gentoolkit.formatters import CpvValueWrapper -from gentoolkit.cpv import split_cpv - -def nl(lines=1): - """small utility function to print blank lines - - @type lines: integer - @param lines: optional number of blank lines to print - default = 1 - """ - print(('\n' * lines)) - -class AnalysisPrinter(CpvValueWrapper): - """Printing functions""" - def __init__(self, target, verbose=True, references=None, key_width=1, width=None): - """@param references: list of accepted keywords or - the system use flags - """ - self.references = references - self.key_width = key_width - self.width = width - CpvValueWrapper.__init__(self, cpv_width=key_width, width=width) - self.set_target(target, verbose) - - def set_target(self, target, verbose=True): - if target in ["use"]: - if verbose: - self.print_fn = self.print_use_verbose - else: - self.print_fn = self.print_use_quiet - self._format_key = self._format_use_keyword - elif target in ["keywords"]: - if verbose: - self.print_fn = self.print_keyword_verbose - else: - self.print_fn = self.print_keyword_quiet - self._format_key = self._format_use_keyword - elif target in ["packages"]: - if verbose: - self.print_fn = self.print_pkg_verbose - else: - self.print_fn = self.print_pkg_quiet - self._format_key = self._format_pkg - - def __call__(self, key, active, data): - self._format_key(key, active, data) - - def _format_use_keyword(self, key, active, pkgs): - """Determines the stats for key, formats it and - calls the pre-determined print function - """ - occurred = str(len(pkgs)) - if active in ["-", "~"]: - _key = active + key - else: - _key = key - if _key in self.references: - default = "default" - else: - default = "......." - count = ' '*(5-len(occurred)) + occurred - pkgs.sort() - self.print_fn(key, active, default, count, pkgs) - - @staticmethod - def print_use_verbose(key, active, default, count, pkgs): - """Verbosely prints a set of use flag info. including the pkgs - using them. - """ - _pkgs = pkgs[:] - if active in ["+", "-"]: - _key = pp.useflag((active+key), active=="+") - else: - _key = (" " + key) - cpv = _pkgs.pop(0) - print(_key,'.'*(35-len(key)), default, pp.number(count), pp.cpv(cpv)) - while _pkgs: - cpv = _pkgs.pop(0) - print(' '*52 + pp.cpv(cpv)) - - # W0613: *Unused argument %r* - # pylint: disable-msg=W0613 - @staticmethod - def print_use_quiet(key, active, default, count, pkgs): - """Quietly prints a subset set of USE flag info.. - """ - if active in ["+", "-"]: - _key = pp.useflag((active+key), active=="+") - else: - _key = (" " + key) - print(_key,'.'*(35-len(key)), default, pp.number(count)) - - @staticmethod - def print_keyword_verbose(key, stability, default, count, pkgs): - """Verbosely prints a set of keywords info. including the pkgs - using them. - """ - _pkgs = pkgs[:] - _key = (pp.keyword((stability+key),stable=(stability==" "), - hard_masked=stability=="-")) - cpv = _pkgs.pop(0) - print(_key,'.'*(20-len(key)), default, pp.number(count), pp.cpv(cpv)) - while _pkgs: - cpv = _pkgs.pop(0) - print(' '*37 + pp.cpv(cpv)) - - # W0613: *Unused argument %r* - # pylint: disable-msg=W0613 - @staticmethod - def print_keyword_quiet(key, stability, default, count, pkgs): - """Quietly prints a subset set of USE flag info.. - """ - _key = (pp.keyword((stability+key), stable=(stability==" "), - hard_masked=stability=="-")) - print(_key,'.'*(20-len(key)), default, pp.number(count)) - - # W0613: *Unused argument %r* - # pylint: disable-msg=W0613 - def _format_pkg(self, key, active, flags): - """Determines the stats for key, formats it and - calls the pre-determined print function - """ - (plus, minus, cleaned) = flags - _plus = [] - _minus = [] - _cleaned = [] - for flag in plus: - _flag = flag.strip() - if _flag: - _plus.append(_flag) - for flag in minus: - _flag = flag.strip() - if _flag: - _minus.append(_flag) - for flag in cleaned: - _flag = flag.strip() - if _flag: - _cleaned.append(_flag) - #print("cpv=", key, "_plus=", _plus, "_minus=", _minus) - self.print_fn(key, (plus, minus, cleaned)) - - def print_pkg_verbose(self, cpv, flags): - """Verbosely prints the pkg's use flag info. - """ - (plus, minus, unset) = flags - _flags = [] - for flag in plus: - _flags.append(pp.useflag((flag), True)) - for flag in minus: - _flags.append(pp.useflag(('-' + flag), False)) - for flag in unset: - _flags.append(pp.globaloption('-' + flag)) - - print(self._format_values(cpv, ", ".join(_flags))) - - - def print_pkg_quiet(self, cpv, flags): - """Verbosely prints the pkg's use flag info. - """ - (plus, minus, unset) = flags - _flags = [] - for flag in plus: - _flags.append(pp.useflag((flag), True)) - for flag in minus: - _flags.append(pp.useflag(('-'+flag), False)) - for flag in unset: - _flags.append(pp.globaloption('-' + flag)) - - print(self._format_values(cpv, ", ".join(_flags))) - - -class RebuildPrinter(CpvValueWrapper): - """Output functions""" - def __init__(self, target, pretend=True, exact=False, - slot=False, key_width=1, width=None): - """@param references: list of accepted keywords or - the system use flags - """ - self.target = target - self.set_target(target) - self.pretend = pretend - CpvValueWrapper.__init__(self, cpv_width=key_width, width=width) - if pretend: - self.spacer = ' ' - self.init_indent = len(self.spacer) - else: - self.spacer = '' - self.exact = exact - self.slot = slot - self.data = {} - - - def set_target(self, target): - if target in ["use"]: - self.print_fn = self.print_use - elif target in ["keywords"]: - self.print_fn = self.print_keyword - elif target in ["unmask"]: - self.print_fn = self.print_mask - self.lines = [self.header()] - - - def __call__(self, key, values, cp_count): - if self.target in ["keywords", "use"]: - self._format_atoms(key, values, cp_count) - else: - self._format_key(key, values) - - - def _format_key(self, key, values): - """Determines the stats for key, formats it and - calls the pre-determined print function - """ - if self.exact: - _key = "=" + key - else: - parts = split_cpv(key) - _key = '/'.join(parts[:2]) - values.sort() - self.data[_key] = values - self.print_fn( _key, values) - - def print_use(self, key, atom=None, values=None): - """Prints a USE flag string. - """ - if atom and not values: - values = atom.use - if self.pretend: - flags = [] - for flag in values: - flags.append(pp.useflag(flag, (flag[0] != '-'))) - print(self._format_values(self.spacer+key, ' '.join(flags))) - else: - line = ' '.join([key, ' '.join(values)]) - self.lines.append(line) - - def _format_atoms(self, key, atoms, count): - """Determines if there are more than one atom in the values and - calls the predetermined print function for each atom. - """ - #print("_format_atoms(),", key, atoms) - if self.exact: - for atom in atoms: - self.print_fn(str(atom), atom=atom) - return - #print("_format_atoms(), count =", count) - if self.slot or count > 1: - for atom in atoms: - _key = str(atom.cp) + ":" + atom.slot - self.print_fn(_key, atom=atom) - else: - for atom in atoms: - _key = str(atom.cp) - self.print_fn(_key, atom=atom) - return - - def print_keyword(self, key, atom=None, keyword=None): - """prints a pkg key and a keyword""" - #print("print_keyword(),", key, keyword) - if atom and not keyword: - keyword = atom.keyword - if self.pretend: - print(self._format_values(key, keyword)) - else: - line = ' '.join([key, keyword]) - self.lines.append(line) - - - def print_unmask(self): - pass - - def header(self): - """Generates a file header - """ - - h=("# This package.%s file was generated by " - %self.target + - "gentoolkit's 'analyse rebuild' module\n" - "# Date: " + time.asctime() + "\n" - ) - return h diff --git a/pym/gentoolkit/analyse/rebuild.py b/pym/gentoolkit/analyse/rebuild.py deleted file mode 100644 index 091df3a..0000000 --- a/pym/gentoolkit/analyse/rebuild.py +++ /dev/null @@ -1,369 +0,0 @@ -#!/usr/bin/python -# -# Copyright 2010 Brian Dolbec -# Copyright(c) 2010, Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 -# - - -"""Provides a rebuild file of USE flags or keywords used and by -what packages according to the Installed package database""" - - -from __future__ import print_function - - -import os -import sys - -import gentoolkit -from gentoolkit.dbapi import PORTDB, VARDB -from gentoolkit.analyse.base import ModuleBase -from gentoolkit import pprinter as pp -from gentoolkit.analyse.lib import (get_installed_use, get_flags, FlagAnalyzer, - KeywordAnalyser) -from gentoolkit.flag import reduce_flags -from gentoolkit.analyse.output import RebuildPrinter -from gentoolkit.atom import Atom - - -import portage - - -def cpv_all_diff_use( - cpvs=None, - system_flags=None, - # override-able for testing - _get_flags=get_flags, - _get_used=get_installed_use - ): - """Data gathering and analysis function determines - the difference between the current default USE flag settings - and the currently installed pkgs recorded USE flag settings - - @type cpvs: list - @param cpvs: optional list of [cat/pkg-ver,...] to analyse or - defaults to entire installed pkg db - @type: system_flags: list - @param system_flags: the current default USE flags as defined - by portage.settings["USE"].split() - @type _get_flags: function - @param _get_flags: ovride-able for testing, - defaults to gentoolkit.analyse.lib.get_flags - @param _get_used: ovride-able for testing, - defaults to gentoolkit.analyse.lib.get_installed_use - @rtype dict. {cpv:['flag1', '-flag2',...]} - """ - if cpvs is None: - cpvs = VARDB.cpv_all() - cpvs.sort() - data = {} - cp_counts = {} - # pass them in to override for tests - flags = FlagAnalyzer(system_flags, - filter_defaults=True, - target="USE", - _get_flags=_get_flags, - _get_used=get_installed_use - ) - for cpv in cpvs: - plus, minus, unset = flags.analyse_cpv(cpv) - atom = Atom("="+cpv) - atom.slot = VARDB.aux_get(atom.cpv, ["SLOT"])[0] - for flag in minus: - plus.add("-"+flag) - if len(plus): - if atom.cp not in data: - data[atom.cp] = [] - if atom.cp not in cp_counts: - cp_counts[atom.cp] = 0 - atom.use = list(plus) - data[atom.cp].append(atom) - cp_counts[atom.cp] += 1 - return data, cp_counts - - -def cpv_all_diff_keywords( - cpvs=None, - system_keywords=None, - use_portage=False, - # override-able for testing - keywords=portage.settings["ACCEPT_KEYWORDS"], - analyser = None - ): - """Analyse the installed pkgs 'keywords' for difference from ACCEPT_KEYWORDS - - @param cpvs: optional list of [cat/pkg-ver,...] to analyse or - defaults to entire installed pkg db - @param system_keywords: list of the system keywords - @param keywords: user defined list of keywords to check and report on - or reports on all relevant keywords found to have been used. - @param _get_kwds: overridable function for testing - @param _get_used: overridable function for testing - @rtype dict. {keyword:{"stable":[cat/pkg-ver,...], - "testing":[cat/pkg-ver,...]} - """ - if cpvs is None: - cpvs = VARDB.cpv_all() - keyword_users = {} - cp_counts = {} - for cpv in cpvs: - if cpv.startswith("virtual"): - continue - if use_portage: - keyword = analyser.get_inst_keyword_cpv(cpv) - else: - pkg = Package(cpv) - keyword = analyser.get_inst_keyword_pkg(pkg) - #print "returned keyword =", cpv, keyword, keyword[0] - key = keyword[0] - if key in ["~", "-"] and keyword not in system_keywords: - atom = Atom("="+cpv) - if atom.cp not in keyword_users: - keyword_users[atom.cp] = [] - if atom.cp not in cp_counts: - cp_counts[atom.cp] = 0 - if key in ["~"]: - atom.keyword = keyword - atom.slot = VARDB.aux_get(atom.cpv, ["SLOT"])[0] - keyword_users[atom.cp].append(atom) - cp_counts[atom.cp] += 1 - elif key in ["-"]: - #print "adding cpv to missing:", cpv - atom.keyword = "**" - atom.slot = VARDB.aux_get(atom.cpv, ["SLOT"])[0] - keyword_users[atom.cp].append(atom) - cp_counts[atom.cp] += 1 - return keyword_users, cp_counts - - -class Rebuild(ModuleBase): - """Installed db analysis tool to query the installed databse - and produce/output stats for USE flags or keywords/mask. - The 'rebuild' action output is in the form suitable for file type output - to create a new package.use, package.keywords, package.unmask - type files in the event of needing to rebuild the - /etc/portage/* user configs - """ - def __init__(self): - ModuleBase.__init__(self) - self.module_name = "rebuild" - self.options = { - "use": False, - "keywords": False, - "unmask": False, - "verbose": False, - "quiet": False, - "exact": False, - "pretend": False, - "prefix": False, - "portage": True, - "slot": False - #"unset": False - } - self.module_opts = { - "-p": ("pretend", "boolean", True), - "--pretend": ("pretend", "boolean", True), - "-e": ("exact", "boolean", True), - "--exact": ("exact", "boolean", True), - "-s": ("slot", "boolean", True), - "--slot": ("slot", "boolean", True), - "-v": ("verbose", "boolean", True), - "--verbose": ("verbose", "boolean", True), - } - self.formatted_options = [ - (" -h, --help", "Outputs this useage message"), - (" -p, --pretend", "Does not actually create the files."), - (" ", "It directs the outputs to the screen"), - (" -e, --exact", "will atomize the package with a"), - (" ", "leading '=' and include the version"), - (" -s, --slot", "will atomize the package with a"), - (" ", "leading '=' and include the slot") - ] - self.formatted_args = [ - (" use", - "causes the action to analyse the installed packages USE flags"), - (" keywords", - "causes the action to analyse the installed packages keywords"), - (" unmask", - "causes the action to analyse the installed packages " + \ - "current mask status") - ] - self.short_opts = "hepsv" - self.long_opts = ("help", "exact", "pretend", "slot", "verbose") - self.need_queries = True - self.arg_spec = "TargetSpec" - self.arg_options = ['use', 'keywords', 'unmask'] - self.arg_option = False - self.warning = ( - " CAUTION", - "This is beta software and some features/options are incomplete,", - "some features may change in future releases includig its name.", - "The file generated is saved in your home directory", - "Feedback will be appreciated, http://bugs.gentoo.org") - - - - def run(self, input_args, quiet=False): - """runs the module - - @param input_args: input arguments to be parsed - """ - self.options['quiet'] = quiet - query = self.main_setup(input_args) - query = self.validate_query(query) - if query in ["use"]: - self.rebuild_use() - elif query in ["keywords"]: - self.rebuild_keywords() - elif query in ["unmask"]: - self.rebuild_unmask() - - - def rebuild_use(self): - if not self.options["quiet"]: - print() - print(" -- Scanning installed packages for USE flag settings that") - print(" do not match the default settings") - system_use = portage.settings["USE"].split() - output = RebuildPrinter( - "use", self.options["pretend"], self.options["exact"], - self.options['slot']) - pkgs, cp_counts = cpv_all_diff_use(system_flags=system_use) - pkg_count = len(pkgs) - if self.options["verbose"]: - print() - print((pp.emph(" -- Found ") + pp.number(str(pkg_count)) + - pp.emph(" packages that need entries"))) - #print pp.emph(" package.use to maintain their current setting") - if pkgs: - pkg_keys = sorted(pkgs) - #print len(pkgs) - if self.options["pretend"] and not self.options["quiet"]: - print() - print(pp.globaloption( - " -- These are the installed packages & use flags " + - "that were detected")) - print(pp.globaloption(" to need use flag settings other " + - "than the defaults.")) - print() - elif not self.options["quiet"]: - print(" -- preparing pkgs for file entries") - for pkg in pkg_keys: - output(pkg, pkgs[pkg], cp_counts[pkg]) - if self.options['verbose']: - message = (pp.emph(" ") + - pp.number(str(pkg_count)) + - pp.emph(" different packages")) - print() - print(pp.globaloption(" -- Totals")) - print(message) - #print - #unique = list(unique_flags) - #unique.sort() - #print unique - if not self.options["pretend"]: - filepath = os.path.expanduser('~/package.use.test') - self.save_file(filepath, output.lines) - - def rebuild_keywords(self): - print("Module action not yet available") - print() - """This will scan the installed packages db and analyse the - keywords used for installation and produce a report on them. - """ - system_keywords = portage.settings["ACCEPT_KEYWORDS"].split() - output = RebuildPrinter( - "keywords", self.options["pretend"], self.options["exact"], - self.options['slot']) - arch = portage.settings["ARCH"] - if self.options["prefix"]: - # build a new keyword for testing - system_keywords = "~" + arch + "-linux" - if self.options["verbose"] or self.options["prefix"]: - print("Current system ARCH =", arch) - print("Current system ACCEPT_KEYWORDS =", system_keywords) - self.analyser = KeywordAnalyser( arch, system_keywords, VARDB) - #self.analyser.set_order(portage.settings["USE"].split()) - # only for testing - test_use = portage.settings["USE"].split() - if self.options['prefix'] and 'prefix' not in test_use: - print("REBUILD_KEYWORDS() 'prefix' flag not found in system", - "USE flags!!! appending for testing") - print() - test_use.append('prefix') - self.analyser.set_order(test_use) - # /end testing - - cpvs = VARDB.cpv_all() - #print "Total number of installed ebuilds =", len(cpvs) - pkgs, cp_counts = cpv_all_diff_keywords( - cpvs=cpvs, - system_keywords=system_keywords, - use_portage=self.options['portage'], - analyser = self.analyser - ) - #print([pkgs[p][0].cpv for p in pkgs]) - if pkgs: - pkg_keys = sorted(pkgs) - #print(len(pkgs)) - if self.options["pretend"] and not self.options["quiet"]: - print() - print(pp.globaloption( - " -- These are the installed packages & keywords " + - "that were detected")) - print(pp.globaloption(" to need keyword settings other " + - "than the defaults.")) - print() - elif not self.options["quiet"]: - print(" -- preparing pkgs for file entries") - for pkg in pkg_keys: - output(pkg, pkgs[pkg], cp_counts[pkg]) - if not self.options['quiet']: - if self.analyser.mismatched: - print("_________________________________________________") - print(("The following packages were found to have a \n" + - "different recorded ARCH than the current system ARCH")) - for cpv in self.analyser.mismatched: - print("\t", pp.cpv(cpv)) - print("===================================================") - print("Total number of entries in report =", - pp.output.red(str(len(pkg_keys)))) - if self.options["verbose"]: - print("Total number of installed ebuilds =", - pp.output.red(str(len(cpvs)))) - print() - if not self.options["pretend"]: - filepath = os.path.expanduser('~/package.keywords.test') - self.save_file(filepath, output.lines) - - - def rebuild_unmask(self): - print("Module action not yet available") - print() - - - def save_file(self, filepath, data): - """Writes the data to the file determined by filepath - - @param filepath: string. eg. '/path/to/filename' - @param data: list of lines to write to filepath - """ - if not self.options["quiet"]: - print(' - Saving file: %s' %filepath) - with open(filepath, "w") as output: - output.write('\n'.join(data)) - print(" - Done") - - -def main(input_args): - """Common starting method by the analyse master - unless all modules are converted to this class method. - - @param input_args: input args as supplied by equery master module. - """ - query_module = Rebuild() - query_module.run(input_args, gentoolkit.CONFIG['quiet']) - -# vim: set ts=4 sw=4 tw=79: - diff --git a/pym/gentoolkit/enalyze/__init__.py b/pym/gentoolkit/enalyze/__init__.py new file mode 100644 index 0000000..b43a82c --- /dev/null +++ b/pym/gentoolkit/enalyze/__init__.py @@ -0,0 +1,128 @@ +#!/usr/bin/python +# +# Copyright 2010 Brian Dolbec +# Copyright(c) 2010, Gentoo Foundation +# Copyright 2003-2004 Karl Trygve Kalleberg +# Licensed under the GNU General Public License, v2 +# +# $Header: $ + +"""Gentoo's installed packages analysis and repair tool""" + + +# Move to Imports section after Python 2.6 is stable + + +__docformat__ = 'epytext' +# version is dynamically set by distutils sdist +__version__ = "svn" +__productname__ = "enalyze" +__authors__ = ( + 'Brian Dolbec, ' + +) + +# make an exportable copy of the info for help output +MODULE_INFO = { + "__docformat__": __docformat__, + "__doc__": __doc__, + "__version__": __version__, + "__productname__": __productname__, + "__authors__": __authors__ + +} + +import errno +import sys +import time +from getopt import getopt, GetoptError + +import portage + +import gentoolkit as gen +from gentoolkit import errors +from gentoolkit import pprinter as pp +from gentoolkit.base import (initialize_configuration, split_arguments, + parse_global_options, print_help) +from gentoolkit.formatters import format_options + + +NAME_MAP = { + 'a': 'analyze', + 'r': 'rebuild' +} + +FORMATTED_OPTIONS = ( + (" (a)nalyze", + "analyzes the installed PKG database USE flag or keyword useage"), + (" (r)ebuild", + "analyzes the Installed PKG database and generates files suitable"), + (" ", + "to replace corrupted or missing /etc/portage/package.* files") + ) + +def expand_module_name(module_name): + """Returns one of the values of NAME_MAP or raises KeyError""" + + if module_name == 'list': + # list is a Python builtin type, so we must rename our module + return 'list_' + elif module_name in NAME_MAP.values(): + return module_name + else: + return NAME_MAP[module_name] + + +def main(): + """Parse input and run the program.""" + + short_opts = "hqCNV" + long_opts = ( + 'help', 'quiet', 'nocolor', 'no-color', 'no-pipe', 'version', 'debug' + ) + + initialize_configuration() + + try: + global_opts, args = getopt(sys.argv[1:], short_opts, long_opts) + except GetoptError as err: + sys.stderr.write(" \n") + sys.stderr.write(pp.error("Global %s\n" % err)) + print_help(MODULE_INFO, FORMATTED_OPTIONS, with_description=False) + sys.exit(2) + + # Parse global options + need_help = parse_global_options(global_opts, args, MODULE_INFO, FORMATTED_OPTIONS) + + if gen.CONFIG['quiet']: + gen.CONFIG['verbose'] = False + + try: + module_name, module_args = split_arguments(args) + except IndexError: + print_help(MODULE_INFO, FORMATTED_OPTIONS) + sys.exit(2) + + if need_help: + module_args.append('--help') + + try: + expanded_module_name = expand_module_name(module_name) + except KeyError: + sys.stderr.write(pp.error("Unknown module '%s'" % module_name)) + print_help(MODULE_INFO, FORMATTED_OPTIONS, with_description=False) + sys.exit(2) + + try: + loaded_module = __import__( + expanded_module_name, globals(), locals(), [], -1 + ) + loaded_module.main(module_args) + except portage.exception.AmbiguousPackageName as err: + raise errors.GentoolkitAmbiguousPackage(err.args[0]) + except IOError as err: + if err.errno != errno.EPIPE: + raise + +if __name__ == '__main__': + main() diff --git a/pym/gentoolkit/enalyze/analyze.py b/pym/gentoolkit/enalyze/analyze.py new file mode 100644 index 0000000..180865d --- /dev/null +++ b/pym/gentoolkit/enalyze/analyze.py @@ -0,0 +1,448 @@ +#!/usr/bin/python +# +# Copyright 2010 Brian Dolbec +# Copyright(c) 2010, Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# + +"""Provides a breakdown list of USE flags or keywords used and by +what packages according to the Installed package database""" + +from __future__ import print_function + +import sys + +import gentoolkit +from gentoolkit.dbapi import PORTDB, VARDB +from gentoolkit.enalyze.base import ModuleBase +from gentoolkit import pprinter as pp +from gentoolkit.flag import get_installed_use, get_flags +from gentoolkit.enalyze.lib import FlagAnalyzer, KeywordAnalyser +from gentoolkit.enalyze.output import nl, AnalysisPrinter +from gentoolkit.package import Package +from gentoolkit.helpers import get_installed_cpvs + +import portage + + +def gather_flags_info( + cpvs=None, + system_flags=None, + include_unset=False, + target="USE", + use_portage=False, + # override-able for testing + _get_flags=get_flags, + _get_used=get_installed_use + ): + """Analyze the installed pkgs USE flags for frequency of use + + @type cpvs: list + @param cpvs: optional list of [cat/pkg-ver,...] to analyze or + defaults to entire installed pkg db + @type: system_flags: list + @param system_flags: the current default USE flags as defined + by portage.settings["USE"].split() + @type include_unset: bool + @param include_unset: controls the inclusion of unset USE flags in the report. + @type target: string + @param target: the environment variable being analyzed + one of ["USE", "PKGUSE"] + @type _get_flags: function + @param _get_flags: ovride-able for testing, + defaults to gentoolkit.enalyze.lib.get_flags + @param _get_used: ovride-able for testing, + defaults to gentoolkit.enalyze.lib.get_installed_use + @rtype dict. {flag:{"+":[cat/pkg-ver,...], "-":[cat/pkg-ver,...], "unset":[]} + """ + if cpvs is None: + cpvs = VARDB.cpv_all() + # pass them in to override for tests + flags = FlagAnalyzer(system_flags, + filter_defaults=False, + target=target, + _get_flags=_get_flags, + _get_used=get_installed_use + ) + flag_users = {} + for cpv in cpvs: + if cpv.startswith("virtual"): + continue + if use_portage: + plus, minus, unset = flags.analyse_cpv(cpv) + else: + pkg = Package(cpv) + plus, minus, unset = flags.analyse_pkg(pkg) + for flag in plus: + if flag in flag_users: + flag_users[flag]["+"].append(cpv) + else: + flag_users[flag] = {"+": [cpv], "-": []} + for flag in minus: + if flag in flag_users: + flag_users[flag]["-"].append(cpv) + else: + flag_users[flag] = {"+":[], "-": [cpv]} + if include_unset: + for flag in unset: + if flag in flag_users: + if "unset" in flag_users[flag]: + flag_users[flag]["unset"].append(cpv) + else: + flag_users[flag]["unset"] = [cpv] + else: + flag_users[flag] = {"+": [], "-": [], "unset": [cpv]} + return flag_users + + +def gather_keywords_info( + cpvs=None, + system_keywords=None, + use_portage=False, + # override-able for testing + keywords=portage.settings["ACCEPT_KEYWORDS"], + analyser = None + ): + """Analyze the installed pkgs 'keywords' for frequency of use + + @param cpvs: optional list of [cat/pkg-ver,...] to analyze or + defaults to entire installed pkg db + @param system_keywords: list of the system keywords + @param keywords: user defined list of keywords to check and report on + or reports on all relevant keywords found to have been used. + @param _get_kwds: overridable function for testing + @param _get_used: overridable function for testing + @rtype dict. {keyword:{"stable":[cat/pkg-ver,...], "testing":[cat/pkg-ver,...]} + """ + if cpvs is None: + cpvs = VARDB.cpv_all() + keyword_users = {} + for cpv in cpvs: + if cpv.startswith("virtual"): + continue + if use_portage: + keyword = analyser.get_inst_keyword_cpv(cpv) + else: + pkg = Package(cpv) + keyword = analyser.get_inst_keyword_pkg(pkg) + #print "returned keyword =", cpv, keyword, keyword[0] + key = keyword[0] + if key in ["~", "-"]: + _kwd = keyword[1:] + if _kwd in keyword_users: + if key in ["~"]: + keyword_users[_kwd]["testing"].append(cpv) + elif key in ["-"]: + #print "adding cpv to missing:", cpv + keyword_users[_kwd]["missing"].append(cpv) + else: + if key in ["~"]: + keyword_users[_kwd] = {"stable": [], + "testing": [cpv], "missing": []} + elif key in ["-"]: + keyword_users[_kwd] = {"stable": [], + "testing": [], "missing": [cpv]} + else: + keyword_users[_kwd] = {"stable": [cpv], + "testing": [], "missing": []} + elif keyword in keyword_users: + keyword_users[keyword]["stable"].append(cpv) + else: + keyword_users[keyword] = { + "stable": [cpv], + "testing": [], + "missing": [] + } + return keyword_users + + +class Analyse(ModuleBase): + """Installed db analysis tool to query the installed databse + and produce/output stats for USE flags or keywords/mask. + The 'rebuild' action output is in the form suitable for file type output + to create a new package.use, package.keywords, package.unmask + type files in the event of needing to rebuild the + /etc/portage/* user configs + """ + def __init__(self): + ModuleBase.__init__(self) + self.module_name = "enalyze" + self.options = { + "flags": False, + "keywords": False, + "packages": False, + "unset": False, + "verbose": False, + "quiet": False, + 'prefix': False, + 'portage': True + } + self.module_opts = { + "-f": ("flags", "boolean", True), + "--flags": ("flags", "boolean", True), + "-k": ("keywords", "boolean", True), + "--keywords": ("keywords", "boolean", True), + "-u": ("unset", "boolean", True), + "--unset": ("unset", "boolean", True), + "-v": ("verbose", "boolean", True), + "--verbose": ("verbose", "boolean", True), + "-p": ("prefix", "boolean", True), + "--prefix": ("prefix", "boolean", True), + "-G": ("portage", "boolean", False), + "--portage": ("portage", "boolean", False), + } + self.formatted_options = [ + (" -h, --help", "Outputs this useage message"), + (" -a, --analyze", + "Action, sets the module to gather data and output the"), + ("", "formatted stats/information to the screen"), + (" -u, --unset", + "Additionally include any unset USE flags and the packages"), + ("", "that could use them"), + (" -v, --verbose", + "Used in the analyze action to output more detailed information"), + (" -p, --prefix", + "Used for testing purposes only, runs report using " + + "a prefix keyword and 'prefix' USE flag"), + #(" -G, --portage", + #"Use portage directly instead of gentoolkit's Package " + + #"object for some operations. Usually a little faster."), + ] + self.formatted_args = [ + (" use", + "Causes the action to analyze the installed packages USE flags"), + (" pkguse", + "Causes the action to analyze the installed packages PKGUSE flags"), + (" ", + "These are flags that have been set in /etc/portage/package.use"), + (" keywords", + "Causes the action to analyze the installed packages keywords"), + (" packages", + "Causes the action to analyze the installed packages and the"), + (" ", + "USE flags they were installed with"), + ] + self.short_opts = "huvpG" + self.long_opts = ("help", "unset", "verbose", "prefix") #, "portage") + self.need_queries = True + self.arg_spec = "Target" + self.arg_options = ['use', 'pkguse','keywords', 'packages'] + self.arg_option = False + self.warning = ( + " CAUTION", + "This is beta software and some features/options are incomplete,", + "some features may change in future releases includig its name.", + "Feedback will be appreciated, http://bugs.gentoo.org") + + + def run(self, input_args, quiet=False): + """runs the module + + @param input_args: input arguments to be parsed + """ + query = self.main_setup(input_args) + query = self.validate_query(query) + self.set_quiet(quiet) + if query in ["use", "pkguse"]: + self.analyse_flags(query) + elif query in ["keywords"]: + self.analyse_keywords() + elif query in ["packages"]: + self.analyse_packages() + + def analyse_flags(self, target): + """This will scan the installed packages db and analyze the + USE flags used for installation and produce a report on how + they were used. + + @type target: string + @param target: the target to be analyzed, one of ["use", "pkguse"] + """ + system_use = portage.settings["USE"].split() + self.printer = AnalysisPrinter( + "use", + self.options["verbose"], + system_use) + if self.options["verbose"]: + cpvs = VARDB.cpv_all() + #cpvs = get_installed_cpvs() + #print "Total number of installed ebuilds =", len(cpvs) + flag_users = gather_flags_info(cpvs, system_use, + self.options["unset"], target=target.upper(), + use_portage=self.options['portage']) + else: + cpvs = get_installed_cpvs() + flag_users = gather_flags_info(cpvs, system_flags=system_use, + include_unset=self.options["unset"], target=target.upper(), + use_portage=self.options['portage']) + #print flag_users + flag_keys = sorted(flag_users) + if self.options["verbose"]: + print(" Flag System #pkgs cat/pkg-ver") + blankline = nl + elif not self.options['quiet']: + print(" Flag System #pkgs") + blankline = lambda: None + for flag in flag_keys: + flag_pos = flag_users[flag]["+"] + if len(flag_pos): + self.printer(flag, "+", flag_pos) + #blankline() + flag_neg = flag_users[flag]["-"] + if len(flag_neg): + self.printer(flag, "-", flag_neg) + #blankline() + if "unset" in flag_users[flag] and flag_users[flag]["unset"]: + flag_unset = flag_users[flag]["unset"] + self.printer(flag, "unset", flag_unset) + #blankline() + if not self.options['quiet']: + print("===================================================") + print("Total number of flags in report =", + pp.output.red(str(len(flag_keys)))) + if self.options["verbose"]: + print("Total number of installed ebuilds =", + pp.output.red(str(len([x for x in cpvs])))) + print() + + + def analyse_keywords(self, keywords=None): + """This will scan the installed packages db and analyze the + keywords used for installation and produce a report on them. + """ + print() + system_keywords = portage.settings["ACCEPT_KEYWORDS"] + arch = portage.settings["ARCH"] + if self.options["prefix"]: + # build a new keyword for testing + system_keywords = "~" + arch + "-linux" + if self.options["verbose"] or self.options["prefix"]: + print("Current system ARCH =", arch) + print("Current system ACCEPT_KEYWORDS =", system_keywords) + system_keywords = system_keywords.split() + self.printer = AnalysisPrinter( + "keywords", + self.options["verbose"], + system_keywords) + self.analyser = KeywordAnalyser( arch, system_keywords, VARDB) + #self.analyser.set_order(portage.settings["USE"].split()) + # only for testing + test_use = portage.settings["USE"].split() + if self.options['prefix'] and 'prefix' not in test_use: + print("ANALYSE_KEYWORDS() 'prefix' flag not found in system", + "USE flags!!! appending for testing") + print() + test_use.append('prefix') + self.analyser.set_order(test_use) + # /end testing + + if self.options["verbose"]: + cpvs = VARDB.cpv_all() + #print "Total number of installed ebuilds =", len(cpvs) + keyword_users = gather_keywords_info( + cpvs=cpvs, + system_keywords=system_keywords, + use_portage=self.options['portage'], + keywords=keywords, analyser = self.analyser + ) + blankline = nl + else: + keyword_users = gather_keywords_info( + system_keywords=system_keywords, + use_portage=self.options['portage'], + keywords=keywords, + analyser = self.analyser + ) + blankline = lambda: None + #print keyword_users + keyword_keys = sorted(keyword_users) + if self.options["verbose"]: + print(" Keyword System #pkgs cat/pkg-ver") + elif not self.options['quiet']: + print(" Keyword System #pkgs") + for keyword in keyword_keys: + kwd_stable = keyword_users[keyword]["stable"] + if len(kwd_stable): + self.printer(keyword, " ", kwd_stable) + blankline() + kwd_testing = keyword_users[keyword]["testing"] + if len(kwd_testing): + self.printer(keyword, "~", kwd_testing) + blankline() + kwd_missing = keyword_users[keyword]["missing"] + if len(kwd_missing): + self.printer(keyword, "-", kwd_missing) + blankline + if not self.options['quiet']: + if self.analyser.mismatched: + print("_________________________________________________") + print(("The following packages were found to have a \n" + + "different recorded ARCH than the current system ARCH")) + for cpv in self.analyser.mismatched: + print("\t", pp.cpv(cpv)) + print("===================================================") + print("Total number of keywords in report =", + pp.output.red(str(len(keyword_keys)))) + if self.options["verbose"]: + print("Total number of installed ebuilds =", + pp.output.red(str(len(cpvs)))) + print() + + + def analyse_packages(self): + """This will scan the installed packages db and analyze the + USE flags used for installation and produce a report. + + @type target: string + @param target: the target to be analyzed, one of ["use", "pkguse"] + """ + system_use = portage.settings["USE"].split() + if self.options["verbose"]: + cpvs = VARDB.cpv_all() + key_width = 45 + else: + cpvs = get_installed_cpvs() + key_width = 1 + + self.printer = AnalysisPrinter( + "packages", + self.options["verbose"], + key_width=key_width) + + cpvs = sorted(cpvs) + flags = FlagAnalyzer( + system=system_use, + filter_defaults=False, + target="USE" + ) + + if self.options["verbose"]: + print(" cat/pkg-ver USE Flags") + # "app-emulation/emul-linux-x86-sdl-20100915 ...." + blankline = nl + elif not self.options['quiet']: + print(" cat/pkg-ver USE Flags") + blankline = lambda: None + for cpv in cpvs: + (flag_plus, flag_neg, unset) = flags.analyse_cpv(cpv) + if self.options["unset"]: + self.printer(cpv, "", (flag_plus, flag_neg, unset)) + else: + self.printer(cpv, "", (flag_plus, flag_neg, [])) + if not self.options['quiet']: + print("===================================================") + print("Total number of installed ebuilds =", + pp.output.red(str(len([x for x in cpvs])))) + print() + + +def main(input_args): + """Common starting method by the analyze master + unless all modules are converted to this class method. + + @param input_args: input args as supplied by equery master module. + """ + query_module = Analyse() + query_module.run(input_args, gentoolkit.CONFIG['quiet']) + +# vim: set ts=4 sw=4 tw=79: diff --git a/pym/gentoolkit/enalyze/base.py b/pym/gentoolkit/enalyze/base.py new file mode 100644 index 0000000..6622704 --- /dev/null +++ b/pym/gentoolkit/enalyze/base.py @@ -0,0 +1,148 @@ +# Copyright(c) 2009, Gentoo Foundation +# +# Copyright 2010 Brian Dolbec +# Copyright(c) 2010, Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# +# $Header: $ + +"""Enalyze Base Module class to hold common module operation functions +""" + +from __future__ import print_function + +__docformat__ = 'epytext' + + +import errno +import sys +import time +from getopt import gnu_getopt, GetoptError + +import gentoolkit.pprinter as pp +from gentoolkit.formatters import format_options +from gentoolkit.base import mod_usage +from gentoolkit import CONFIG + +class ModuleBase(object): + """Enalyze base module class to parse module options print module help, etc..""" + + def __init__(self): + self.module_name = None + self.options = {} + self.formatted_options = None + self.short_opts = None + self.long_opts = None + self.module_opts = {} + self.warning = None + self.need_queries = True + self.saved_verbose = None + + + def print_help(self, with_description=True): + """Print description, usage and a detailed help message. + + @type with_description: bool + @param with_description: if true, print module's __doc__ string + """ + + if with_description: + print() + print(__doc__.strip()) + print() + if self.warning: + print() + for line in self.warning: + sys.stderr.write(pp.warn(line)) + print() + print(mod_usage(mod_name=self.module_name, arg=self.arg_spec, optional=self.arg_option)) + print() + print(pp.command("options")) + print(format_options( self.formatted_options )) + if self.formatted_args: + print() + print(pp.command(self.arg_spec)) + print(format_options(self.formatted_args)) + print() + + def parse_module_options(self, module_opts): + """Parse module options and update self.options""" + + opts = (x[0] for x in module_opts) + posargs = (x[1] for x in module_opts) + for opt, posarg in zip(opts, posargs): + if opt in ('-h', '--help'): + self.print_help() + sys.exit(0) + opt_name, opt_type, opt_setting = self.module_opts[opt] + if opt_type == 'boolean': + self.options[opt_name] = opt_setting + elif opt_type == 'int': + if posarg.isdigit(): + val = int(posarg) + else: + print() + err = "Module option %s requires integer (got '%s')" + sys.stdout.write(pp.error(err % (opt,posarg))) + print() + self.print_help(with_description=False) + sys.exit(2) + self.options[opt_name] = val + + def set_quiet(self, quiet): + """sets the class option["quiet"] and option["verbose"] accordingly""" + if quiet == self.options['quiet']: + return + if self.saved_verbose: + # detected a switch + verbose = self.options['verbose'] + self.options['verbose'] = self.saved_verbose + self.saved_verbose = verbose + elif quiet: + self.saved_verbose = self.options['verbose'] + self.options['verbose'] = False + self.options['quiet'] = quiet + return + + def validate_query(self, query, depth=0): + """check that the query meets the modules TargetSpec + If not it attempts to reduce it to a valid TargetSpec + or prints the help message and exits + """ + if depth > 1: + return [] + if len(query) > 1: + query = list(set(self.arg_options).intersection(query)) + #print "reduced query =", query + query = self.validate_query(query, depth+1) + if isinstance(query, list): + query = query[0] + if query not in self.arg_options: + print() + print(pp.error( + "Error starting module. Incorrect or No TargetSpec specified!" + )) + print("query = ", query) + self.print_help() + sys.exit(2) + return query + + + def main_setup(self, input_args): + """Parse input and prepares the program""" + + try: + module_opts, queries = gnu_getopt(input_args, self.short_opts, self.long_opts) + except GetoptError as err: + sys.stderr.write(pp.error("Module %s" % err)) + print() + self.print_help(with_description=False) + sys.exit(2) + self.parse_module_options(module_opts) + if self.need_queries and not queries: + self.print_help() + sys.exit(2) + return queries + + +# vim: set ts=4 sw=4 tw=79: diff --git a/pym/gentoolkit/enalyze/lib.py b/pym/gentoolkit/enalyze/lib.py new file mode 100644 index 0000000..015e23b --- /dev/null +++ b/pym/gentoolkit/enalyze/lib.py @@ -0,0 +1,354 @@ +#!/usr/bin/python +# +# Copyright 2010 Brian Dolbec +# Copyright(c) 2010, Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# + + +"""Provides support functions to enalyze modules""" + +import sys + +from gentoolkit.dbapi import PORTDB, VARDB +from gentoolkit import errors +from gentoolkit.keyword import reduce_keywords +from gentoolkit.flag import (reduce_flags, get_flags, get_all_cpv_use, + filter_flags, get_installed_use, get_iuse) +#from gentoolkit.package import Package + +import portage + + +class FlagAnalyzer(object): + """Specialty functions for analysing an installed package's + USE flags. Can be used for single or mulitple use without + needing to be reset unless the system USE flags are changed. + + @type system: list or set + @param system: the default system USE flags. + @type _get_flags: function + @param _get_flags: Normally defaulted, can be overriden for testing + @type _get_used: function + @param _get_used: Normally defaulted, can be overriden for testing + """ + def __init__(self, + system, + filter_defaults=False, + target="USE", + _get_flags=get_flags, + _get_used=get_installed_use + ): + self.get_flags = _get_flags + self.get_used = _get_used + self.filter_defaults = filter_defaults + self.target = target + self.reset(system) + + def reset(self, system): + """Resets the internal system USE flags and use_expand variables + to the new setting. The use_expand variable is handled internally. + + @type system: list or set + @param system: the default system USE flags. + """ + self.system = set(system) + self.use_expand = portage.settings['USE_EXPAND'].lower().split() + + def analyse_cpv(self, cpv): + """Gets all relavent USE flag info for a cpv and breaks them down + into 3 sets, plus (package.use enabled), minus ( package.use disabled), + unset. + + @param cpv: string. 'cat/pkg-ver' + @rtype tuple of sets + @return (plus, minus, unset) sets of USE flags + """ + installed = set(self.get_used(cpv, self.target)) + iuse = set(reduce_flags(self.get_flags(cpv))) + return self._analyse(installed, iuse) + + def _analyse(self, installed, iuse): + """Analyzes the supplied info and returns the flag settings + that differ from the defaults + + @type installed: set + @param installed: the installed with use flags + @type iuse: set + @param iuse: the current ebuilds IUSE + """ + defaults = self.system.intersection(iuse) + usedflags = iuse.intersection(set(installed)) + if self.filter_defaults: + plus = usedflags.difference(defaults) + else: + plus = usedflags + minus = defaults.difference(usedflags) + unset = iuse.difference(defaults, plus, minus) + cleaned_unset = self.remove_expanding(unset) + return (plus, minus, cleaned_unset) + + def analyse_pkg(self, pkg): + """Gets all relevent USE flag info for a pkg and breaks them down + into 3 sets, plus (package.use enabled), minus ( package.use disabled), + unset. + + @param pkg: gentoolkit.package.Package object + @rtype tuple of sets + @return (plus, minus, unset) sets of USE flags + """ + installed = set(self.pkg_used(pkg)) + print("installed =", installed) + iuse = set(reduce_flags(self.pkg_flags(pkg))) + print("iuse =", iuse) + return self._analyse(installed, iuse) + + def pkg_used(self, pkg): + if self.target == "USE": + return pkg.use().split() + return pkg.environment(self.target).split() + + def pkg_flags(self, pkg): + final_use, use_expand_hidden, usemasked, useforced = \ + get_all_cpv_use(pkg.cpv) + flags = pkg.environment("IUSE", prefer_vdb=False).split() + return filter_flags(flags, use_expand_hidden, usemasked, useforced) + + def redundant(self, cpv, iuse): + """Checks for redundant settings. + future function. Not yet implemented. + """ + pass + + def remove_expanding(self, flags): + """Remove unwanted USE_EXPAND flags + from unset IUSE sets + + @param flags: short list or set of USE flags + @rtype set + @return USE flags + """ + _flags = set(flags) + for expander in self.use_expand: + for flag in flags: + if expander in flag: + _flags.remove(flag) + if not _flags: + break + return _flags + + +class KeywordAnalyser(object): + """Specialty functions for analysing the installed package db for + keyword useage and the packages that used them. + + Note: should be initialized with the internal set_order() before use. + See internal set_order() for more details. + This class of functions can be used for single cpv checks or + used repeatedly for an entire package db. + + @type arch: string + @param arch: the system ARCH setting + @type accept_keywords: list + @param accept_keywords: eg. ['x86', '~x86'] + @type get_aux: function, defaults to: VARDB.aux_get + @param vardb: vardb class of functions, needed=aux_get() + to return => KEYWORDS & USE flags for a cpv + = aux_get(cpv, ["KEYWORDS", "USE"]) + """ + + # parsing order to determine appropriate keyword used for installation + normal_order = ['stable', 'testing', 'prefix', 'testing_prefix', 'missing'] + prefix_order = ['prefix', 'testing_prefix', 'stable', 'testing', 'missing'] + parse_range = list(range(len(normal_order))) + + + def __init__(self, arch, accept_keywords, vardb=VARDB): + self.arch = arch + self.accept_keywords = accept_keywords + self.vardb = vardb + self.prefix = '' + self.parse_order = None + self.check_key = { + 'stable': self._stable, + 'testing': self._testing, + 'prefix': self._prefix, + 'testing_prefix': self._testing_prefix, + 'missing': self._missing + } + self.mismatched = [] + + def determine_keyword(self, keywords, used, cpv): + """Determine the keyword from the installed USE flags and + the KEYWORDS that was used to install a package. + + @param keywords: list of keywords available to install a pkg + @param used: list of USE flalgs recorded for the installed pkg + @rtype: string + @return a keyword or null string + """ + used = set(used) + kwd = None + result = '' + if keywords: + absolute_kwds = reduce_keywords(keywords) + kwd = list(used.intersection(absolute_kwds)) + #if keywords == ['~ppc64']: + #print "Checked keywords for kwd", keywords, used, "kwd =", kwd + if not kwd: + #print "Checking for kwd against portage.archlist" + absolute_kwds = reduce_keywords(keywords) + # check for one against archlist then re-check + kwd = list(absolute_kwds.intersection(portage.archlist)) + #print "determined keyword =", kwd + if len(kwd) == 1: + key = kwd[0] + #print "determined keyword =", key + elif not kwd: + #print "kwd != 1", kwd, cpv + result = self._missing(self.keyword, keywords) + else: # too many, try to narrow them dowm + #print "too many kwd's, trying to match against arch" + _kwd = list(set(kwd).intersection(self.arch)) + key = '' + if _kwd: + #print "found one! :)", _kwd + key = _kwd + else: # try re-running the short list against archlist + #print "Checking kwd for _kwd against portage.archlist" + _kwd = list(set(kwd).intersection(portage.archlist)) + if _kwd and len(_kwd) == 1: + #print "found one! :)", _kwd + key = _kwd[0] + else: + #print " :( didn't work, _kwd =", _kwd, "giving up on:", cpv + result = self._missing(self.keyword, keywords) + i = 0 + while not result and i in self.parse_range: + parsekey = self.parse_order[i] + result = self.check_key[parsekey](key, keywords) + i += 1 + return result + + def _stable(self, key, keywords): + """test for a normal stable keyword""" + if key in keywords: + return key + return '' + + def _testing(self, key, keywords): + """test for a normal testing keyword""" + if ("~" + key) in keywords: + return "~" + key + return '' + + def _prefix(self, key, keywords): + """test for a stable prefix keyword""" + if not self.prefix: + return '' + _key = '-'.join([key, self.prefix]) + if _key in keywords: + #print key, "is in", keywords + return _key + return '' + + def _testing_prefix(self, key, keywords): + """test for a testing prefix keyword""" + if not self.prefix: + return '' + _key = "~" +'-'.join([key, self.prefix]) + if _key in keywords: + #print key, "is in", keywords + return _key + return '' + + def _missing(self, key, keywords): + """generates a missing keyword to return""" + if self.prefix and key != self.keyword: + _key = '-'.join([key, self.prefix]) + else: + _key = '-' + key + #print "_missisng :( _key =", _key + return _key + + def get_inst_keyword_cpv(self, cpv): + """Determines the installed with keyword for cpv + + @type cpv: string + @param cpv: an installed CAT/PKG-VER + @rtype: string + @returns a keyword determined to have been used to install cpv + """ + keywords, used = self.vardb.aux_get(cpv, ["KEYWORDS", "USE"]) + keywords = keywords.split() + used = used.split() + return self._parse(keywords, used, cpv=cpv) + + def get_inst_keyword_pkg(self, pkg): + """Determines the installed with keyword for cpv + + @param pkg: gentoolkit.package.Package object + @rtype: string + @returns a keyword determined to have been used to install cpv + """ + keywords, used = pkg.environment(["KEYWORDS", "USE"], + prefer_vdb=True, fallback=False) + keywords = keywords.split() + used = used.split() + return self._parse(keywords, used, pkg=pkg) + + def _parse(self, keywords, used, pkg=None, cpv=None): + if pkg: + _cpv = pkg.cpv + else: + _cpv = cpv + if not self.parse_order: + self.set_order(used) + keyword = self.keyword + # sanity check + if self.arch not in used: + #print "Found a mismatch = ", cpv, self.arch, used + self.mismatched.append(_cpv) + if keyword in keywords: + #print "keyword", keyword, "is in", keywords + return keyword + elif "~"+keyword in keywords: + #print "~keyword", keyword, "is in", keywords + return "~"+keyword + else: + keyword = self.determine_keyword(keywords, used, _cpv) + if not keyword: + raise errors.GentoolkitUnknownKeyword(_cpv, ' '.join(keywords), used) + return keyword + + def set_order(self, used): + """Used to set the parsing order to determine a keyword + used for installation. + + This is needed due to the way prefix arch's and keywords + work with portage. It looks for the 'prefix' flag. A positive result + sets it to the prefix order and keyword. + + @type used: list + @param used: a list of pkg USE flags or the system USE flags""" + if 'prefix' in used: + #print "SET_ORDER() Setting parse order to prefix" + prefix = None + self.parse_order = self.prefix_order + for key in self.accept_keywords: + #print "SET_ORDER() '"+key+"'" + if '-' in key: + #print "SET_ORDER()found prefix keyword :", key + if self.arch in key: + prefix = key.split('-')[1] + #print "prefix =", prefix + self.prefix = prefix + self.keyword = '-'.join([self.arch, prefix]) + else: + #print "SET_ORDER() Setting parse order to normal" + self.parse_order = self.normal_order + self.keyword = self.arch + #print "SET_ORDER() completed: prefix =", self.prefix, ", keyword =", \ + # self.keyword, "parse order =",self.parse_order + #print + diff --git a/pym/gentoolkit/enalyze/output.py b/pym/gentoolkit/enalyze/output.py new file mode 100644 index 0000000..326ebbc --- /dev/null +++ b/pym/gentoolkit/enalyze/output.py @@ -0,0 +1,297 @@ +#!/usr/bin/python +# +# Copyright(c) 2010, Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# + +"""Provides various output classes and functions for +both screen and file output +""" + +from __future__ import print_function + +import time + +import gentoolkit +from gentoolkit import pprinter as pp +from gentoolkit.formatters import CpvValueWrapper +from gentoolkit.cpv import split_cpv + +def nl(lines=1): + """small utility function to print blank lines + + @type lines: integer + @param lines: optional number of blank lines to print + default = 1 + """ + print(('\n' * lines)) + +class AnalysisPrinter(CpvValueWrapper): + """Printing functions""" + def __init__(self, target, verbose=True, references=None, key_width=1, width=None): + """@param references: list of accepted keywords or + the system use flags + """ + self.references = references + self.key_width = key_width + self.width = width + CpvValueWrapper.__init__(self, cpv_width=key_width, width=width) + self.set_target(target, verbose) + + def set_target(self, target, verbose=True): + if target in ["use"]: + if verbose: + self.print_fn = self.print_use_verbose + else: + self.print_fn = self.print_use_quiet + self._format_key = self._format_use_keyword + elif target in ["keywords"]: + if verbose: + self.print_fn = self.print_keyword_verbose + else: + self.print_fn = self.print_keyword_quiet + self._format_key = self._format_use_keyword + elif target in ["packages"]: + if verbose: + self.print_fn = self.print_pkg_verbose + else: + self.print_fn = self.print_pkg_quiet + self._format_key = self._format_pkg + + def __call__(self, key, active, data): + self._format_key(key, active, data) + + def _format_use_keyword(self, key, active, pkgs): + """Determines the stats for key, formats it and + calls the pre-determined print function + """ + occurred = str(len(pkgs)) + if active in ["-", "~"]: + _key = active + key + else: + _key = key + if _key in self.references: + default = "default" + else: + default = "......." + count = ' '*(5-len(occurred)) + occurred + pkgs.sort() + self.print_fn(key, active, default, count, pkgs) + + @staticmethod + def print_use_verbose(key, active, default, count, pkgs): + """Verbosely prints a set of use flag info. including the pkgs + using them. + """ + _pkgs = pkgs[:] + if active in ["+", "-"]: + _key = pp.useflag((active+key), active=="+") + else: + _key = (" " + key) + cpv = _pkgs.pop(0) + print(_key,'.'*(35-len(key)), default, pp.number(count), pp.cpv(cpv)) + while _pkgs: + cpv = _pkgs.pop(0) + print(' '*52 + pp.cpv(cpv)) + + # W0613: *Unused argument %r* + # pylint: disable-msg=W0613 + @staticmethod + def print_use_quiet(key, active, default, count, pkgs): + """Quietly prints a subset set of USE flag info.. + """ + if active in ["+", "-"]: + _key = pp.useflag((active+key), active=="+") + else: + _key = (" " + key) + print(_key,'.'*(35-len(key)), default, pp.number(count)) + + @staticmethod + def print_keyword_verbose(key, stability, default, count, pkgs): + """Verbosely prints a set of keywords info. including the pkgs + using them. + """ + _pkgs = pkgs[:] + _key = (pp.keyword((stability+key),stable=(stability==" "), + hard_masked=stability=="-")) + cpv = _pkgs.pop(0) + print(_key,'.'*(20-len(key)), default, pp.number(count), pp.cpv(cpv)) + while _pkgs: + cpv = _pkgs.pop(0) + print(' '*37 + pp.cpv(cpv)) + + # W0613: *Unused argument %r* + # pylint: disable-msg=W0613 + @staticmethod + def print_keyword_quiet(key, stability, default, count, pkgs): + """Quietly prints a subset set of USE flag info.. + """ + _key = (pp.keyword((stability+key), stable=(stability==" "), + hard_masked=stability=="-")) + print(_key,'.'*(20-len(key)), default, pp.number(count)) + + # W0613: *Unused argument %r* + # pylint: disable-msg=W0613 + def _format_pkg(self, key, active, flags): + """Determines the stats for key, formats it and + calls the pre-determined print function + """ + (plus, minus, cleaned) = flags + _plus = [] + _minus = [] + _cleaned = [] + for flag in plus: + _flag = flag.strip() + if _flag: + _plus.append(_flag) + for flag in minus: + _flag = flag.strip() + if _flag: + _minus.append(_flag) + for flag in cleaned: + _flag = flag.strip() + if _flag: + _cleaned.append(_flag) + #print("cpv=", key, "_plus=", _plus, "_minus=", _minus) + self.print_fn(key, (plus, minus, cleaned)) + + def print_pkg_verbose(self, cpv, flags): + """Verbosely prints the pkg's use flag info. + """ + (plus, minus, unset) = flags + _flags = [] + for flag in plus: + _flags.append(pp.useflag((flag), True)) + for flag in minus: + _flags.append(pp.useflag(('-' + flag), False)) + for flag in unset: + _flags.append(pp.globaloption('-' + flag)) + + print(self._format_values(cpv, ", ".join(_flags))) + + + def print_pkg_quiet(self, cpv, flags): + """Verbosely prints the pkg's use flag info. + """ + (plus, minus, unset) = flags + _flags = [] + for flag in plus: + _flags.append(pp.useflag((flag), True)) + for flag in minus: + _flags.append(pp.useflag(('-'+flag), False)) + for flag in unset: + _flags.append(pp.globaloption('-' + flag)) + + print(self._format_values(cpv, ", ".join(_flags))) + + +class RebuildPrinter(CpvValueWrapper): + """Output functions""" + def __init__(self, target, pretend=True, exact=False, + slot=False, key_width=1, width=None): + """@param references: list of accepted keywords or + the system use flags + """ + self.target = target + self.set_target(target) + self.pretend = pretend + CpvValueWrapper.__init__(self, cpv_width=key_width, width=width) + if pretend: + self.spacer = ' ' + self.init_indent = len(self.spacer) + else: + self.spacer = '' + self.exact = exact + self.slot = slot + self.data = {} + + + def set_target(self, target): + if target in ["use"]: + self.print_fn = self.print_use + elif target in ["keywords"]: + self.print_fn = self.print_keyword + elif target in ["unmask"]: + self.print_fn = self.print_mask + self.lines = [self.header()] + + + def __call__(self, key, values, cp_count): + if self.target in ["keywords", "use"]: + self._format_atoms(key, values, cp_count) + else: + self._format_key(key, values) + + + def _format_key(self, key, values): + """Determines the stats for key, formats it and + calls the pre-determined print function + """ + if self.exact: + _key = "=" + key + else: + parts = split_cpv(key) + _key = '/'.join(parts[:2]) + values.sort() + self.data[_key] = values + self.print_fn( _key, values) + + def print_use(self, key, atom=None, values=None): + """Prints a USE flag string. + """ + if atom and not values: + values = atom.use + if self.pretend: + flags = [] + for flag in values: + flags.append(pp.useflag(flag, (flag[0] != '-'))) + print(self._format_values(self.spacer+key, ' '.join(flags))) + else: + line = ' '.join([key, ' '.join(values)]) + self.lines.append(line) + + def _format_atoms(self, key, atoms, count): + """Determines if there are more than one atom in the values and + calls the predetermined print function for each atom. + """ + #print("_format_atoms(),", key, atoms) + if self.exact: + for atom in atoms: + self.print_fn(str(atom), atom=atom) + return + #print("_format_atoms(), count =", count) + if self.slot or count > 1: + for atom in atoms: + _key = str(atom.cp) + ":" + atom.slot + self.print_fn(_key, atom=atom) + else: + for atom in atoms: + _key = str(atom.cp) + self.print_fn(_key, atom=atom) + return + + def print_keyword(self, key, atom=None, keyword=None): + """prints a pkg key and a keyword""" + #print("print_keyword(),", key, keyword) + if atom and not keyword: + keyword = atom.keyword + if self.pretend: + print(self._format_values(key, keyword)) + else: + line = ' '.join([key, keyword]) + self.lines.append(line) + + + def print_unmask(self): + pass + + def header(self): + """Generates a file header + """ + + h=("# This package.%s file was generated by " + %self.target + + "gentoolkit's 'enalyze rebuild' module\n" + "# Date: " + time.asctime() + "\n" + ) + return h diff --git a/pym/gentoolkit/enalyze/rebuild.py b/pym/gentoolkit/enalyze/rebuild.py new file mode 100644 index 0000000..f1d7e88 --- /dev/null +++ b/pym/gentoolkit/enalyze/rebuild.py @@ -0,0 +1,369 @@ +#!/usr/bin/python +# +# Copyright 2010 Brian Dolbec +# Copyright(c) 2010, Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# + + +"""Provides a rebuild file of USE flags or keywords used and by +what packages according to the Installed package database""" + + +from __future__ import print_function + + +import os +import sys + +import gentoolkit +from gentoolkit.dbapi import PORTDB, VARDB +from gentoolkit.enalyze.base import ModuleBase +from gentoolkit import pprinter as pp +from gentoolkit.enalyze.lib import (get_installed_use, get_flags, FlagAnalyzer, + KeywordAnalyser) +from gentoolkit.flag import reduce_flags +from gentoolkit.enalyze.output import RebuildPrinter +from gentoolkit.atom import Atom + + +import portage + + +def cpv_all_diff_use( + cpvs=None, + system_flags=None, + # override-able for testing + _get_flags=get_flags, + _get_used=get_installed_use + ): + """Data gathering and analysis function determines + the difference between the current default USE flag settings + and the currently installed pkgs recorded USE flag settings + + @type cpvs: list + @param cpvs: optional list of [cat/pkg-ver,...] to analyze or + defaults to entire installed pkg db + @type: system_flags: list + @param system_flags: the current default USE flags as defined + by portage.settings["USE"].split() + @type _get_flags: function + @param _get_flags: ovride-able for testing, + defaults to gentoolkit.enalyze.lib.get_flags + @param _get_used: ovride-able for testing, + defaults to gentoolkit.enalyze.lib.get_installed_use + @rtype dict. {cpv:['flag1', '-flag2',...]} + """ + if cpvs is None: + cpvs = VARDB.cpv_all() + cpvs.sort() + data = {} + cp_counts = {} + # pass them in to override for tests + flags = FlagAnalyzer(system_flags, + filter_defaults=True, + target="USE", + _get_flags=_get_flags, + _get_used=get_installed_use + ) + for cpv in cpvs: + plus, minus, unset = flags.analyse_cpv(cpv) + atom = Atom("="+cpv) + atom.slot = VARDB.aux_get(atom.cpv, ["SLOT"])[0] + for flag in minus: + plus.add("-"+flag) + if len(plus): + if atom.cp not in data: + data[atom.cp] = [] + if atom.cp not in cp_counts: + cp_counts[atom.cp] = 0 + atom.use = list(plus) + data[atom.cp].append(atom) + cp_counts[atom.cp] += 1 + return data, cp_counts + + +def cpv_all_diff_keywords( + cpvs=None, + system_keywords=None, + use_portage=False, + # override-able for testing + keywords=portage.settings["ACCEPT_KEYWORDS"], + analyser = None + ): + """Analyze the installed pkgs 'keywords' for difference from ACCEPT_KEYWORDS + + @param cpvs: optional list of [cat/pkg-ver,...] to analyze or + defaults to entire installed pkg db + @param system_keywords: list of the system keywords + @param keywords: user defined list of keywords to check and report on + or reports on all relevant keywords found to have been used. + @param _get_kwds: overridable function for testing + @param _get_used: overridable function for testing + @rtype dict. {keyword:{"stable":[cat/pkg-ver,...], + "testing":[cat/pkg-ver,...]} + """ + if cpvs is None: + cpvs = VARDB.cpv_all() + keyword_users = {} + cp_counts = {} + for cpv in cpvs: + if cpv.startswith("virtual"): + continue + if use_portage: + keyword = analyser.get_inst_keyword_cpv(cpv) + else: + pkg = Package(cpv) + keyword = analyser.get_inst_keyword_pkg(pkg) + #print "returned keyword =", cpv, keyword, keyword[0] + key = keyword[0] + if key in ["~", "-"] and keyword not in system_keywords: + atom = Atom("="+cpv) + if atom.cp not in keyword_users: + keyword_users[atom.cp] = [] + if atom.cp not in cp_counts: + cp_counts[atom.cp] = 0 + if key in ["~"]: + atom.keyword = keyword + atom.slot = VARDB.aux_get(atom.cpv, ["SLOT"])[0] + keyword_users[atom.cp].append(atom) + cp_counts[atom.cp] += 1 + elif key in ["-"]: + #print "adding cpv to missing:", cpv + atom.keyword = "**" + atom.slot = VARDB.aux_get(atom.cpv, ["SLOT"])[0] + keyword_users[atom.cp].append(atom) + cp_counts[atom.cp] += 1 + return keyword_users, cp_counts + + +class Rebuild(ModuleBase): + """Installed db analysis tool to query the installed databse + and produce/output stats for USE flags or keywords/mask. + The 'rebuild' action output is in the form suitable for file type output + to create a new package.use, package.keywords, package.unmask + type files in the event of needing to rebuild the + /etc/portage/* user configs + """ + def __init__(self): + ModuleBase.__init__(self) + self.module_name = "rebuild" + self.options = { + "use": False, + "keywords": False, + "unmask": False, + "verbose": False, + "quiet": False, + "exact": False, + "pretend": False, + "prefix": False, + "portage": True, + "slot": False + #"unset": False + } + self.module_opts = { + "-p": ("pretend", "boolean", True), + "--pretend": ("pretend", "boolean", True), + "-e": ("exact", "boolean", True), + "--exact": ("exact", "boolean", True), + "-s": ("slot", "boolean", True), + "--slot": ("slot", "boolean", True), + "-v": ("verbose", "boolean", True), + "--verbose": ("verbose", "boolean", True), + } + self.formatted_options = [ + (" -h, --help", "Outputs this useage message"), + (" -p, --pretend", "Does not actually create the files."), + (" ", "It directs the outputs to the screen"), + (" -e, --exact", "will atomize the package with a"), + (" ", "leading '=' and include the version"), + (" -s, --slot", "will atomize the package with a"), + (" ", "leading '=' and include the slot") + ] + self.formatted_args = [ + (" use", + "causes the action to analyze the installed packages USE flags"), + (" keywords", + "causes the action to analyze the installed packages keywords"), + (" unmask", + "causes the action to analyze the installed packages " + \ + "current mask status") + ] + self.short_opts = "hepsv" + self.long_opts = ("help", "exact", "pretend", "slot", "verbose") + self.need_queries = True + self.arg_spec = "TargetSpec" + self.arg_options = ['use', 'keywords', 'unmask'] + self.arg_option = False + self.warning = ( + " CAUTION", + "This is beta software and some features/options are incomplete,", + "some features may change in future releases includig its name.", + "The file generated is saved in your home directory", + "Feedback will be appreciated, http://bugs.gentoo.org") + + + + def run(self, input_args, quiet=False): + """runs the module + + @param input_args: input arguments to be parsed + """ + self.options['quiet'] = quiet + query = self.main_setup(input_args) + query = self.validate_query(query) + if query in ["use"]: + self.rebuild_use() + elif query in ["keywords"]: + self.rebuild_keywords() + elif query in ["unmask"]: + self.rebuild_unmask() + + + def rebuild_use(self): + if not self.options["quiet"]: + print() + print(" -- Scanning installed packages for USE flag settings that") + print(" do not match the default settings") + system_use = portage.settings["USE"].split() + output = RebuildPrinter( + "use", self.options["pretend"], self.options["exact"], + self.options['slot']) + pkgs, cp_counts = cpv_all_diff_use(system_flags=system_use) + pkg_count = len(pkgs) + if self.options["verbose"]: + print() + print((pp.emph(" -- Found ") + pp.number(str(pkg_count)) + + pp.emph(" packages that need entries"))) + #print pp.emph(" package.use to maintain their current setting") + if pkgs: + pkg_keys = sorted(pkgs) + #print len(pkgs) + if self.options["pretend"] and not self.options["quiet"]: + print() + print(pp.globaloption( + " -- These are the installed packages & use flags " + + "that were detected")) + print(pp.globaloption(" to need use flag settings other " + + "than the defaults.")) + print() + elif not self.options["quiet"]: + print(" -- preparing pkgs for file entries") + for pkg in pkg_keys: + output(pkg, pkgs[pkg], cp_counts[pkg]) + if self.options['verbose']: + message = (pp.emph(" ") + + pp.number(str(pkg_count)) + + pp.emph(" different packages")) + print() + print(pp.globaloption(" -- Totals")) + print(message) + #print + #unique = list(unique_flags) + #unique.sort() + #print unique + if not self.options["pretend"]: + filepath = os.path.expanduser('~/package.use.test') + self.save_file(filepath, output.lines) + + def rebuild_keywords(self): + print("Module action not yet available") + print() + """This will scan the installed packages db and analyze the + keywords used for installation and produce a report on them. + """ + system_keywords = portage.settings["ACCEPT_KEYWORDS"].split() + output = RebuildPrinter( + "keywords", self.options["pretend"], self.options["exact"], + self.options['slot']) + arch = portage.settings["ARCH"] + if self.options["prefix"]: + # build a new keyword for testing + system_keywords = "~" + arch + "-linux" + if self.options["verbose"] or self.options["prefix"]: + print("Current system ARCH =", arch) + print("Current system ACCEPT_KEYWORDS =", system_keywords) + self.analyser = KeywordAnalyser( arch, system_keywords, VARDB) + #self.analyser.set_order(portage.settings["USE"].split()) + # only for testing + test_use = portage.settings["USE"].split() + if self.options['prefix'] and 'prefix' not in test_use: + print("REBUILD_KEYWORDS() 'prefix' flag not found in system", + "USE flags!!! appending for testing") + print() + test_use.append('prefix') + self.analyser.set_order(test_use) + # /end testing + + cpvs = VARDB.cpv_all() + #print "Total number of installed ebuilds =", len(cpvs) + pkgs, cp_counts = cpv_all_diff_keywords( + cpvs=cpvs, + system_keywords=system_keywords, + use_portage=self.options['portage'], + analyser = self.analyser + ) + #print([pkgs[p][0].cpv for p in pkgs]) + if pkgs: + pkg_keys = sorted(pkgs) + #print(len(pkgs)) + if self.options["pretend"] and not self.options["quiet"]: + print() + print(pp.globaloption( + " -- These are the installed packages & keywords " + + "that were detected")) + print(pp.globaloption(" to need keyword settings other " + + "than the defaults.")) + print() + elif not self.options["quiet"]: + print(" -- preparing pkgs for file entries") + for pkg in pkg_keys: + output(pkg, pkgs[pkg], cp_counts[pkg]) + if not self.options['quiet']: + if self.analyser.mismatched: + print("_________________________________________________") + print(("The following packages were found to have a \n" + + "different recorded ARCH than the current system ARCH")) + for cpv in self.analyser.mismatched: + print("\t", pp.cpv(cpv)) + print("===================================================") + print("Total number of entries in report =", + pp.output.red(str(len(pkg_keys)))) + if self.options["verbose"]: + print("Total number of installed ebuilds =", + pp.output.red(str(len(cpvs)))) + print() + if not self.options["pretend"]: + filepath = os.path.expanduser('~/package.keywords.test') + self.save_file(filepath, output.lines) + + + def rebuild_unmask(self): + print("Module action not yet available") + print() + + + def save_file(self, filepath, data): + """Writes the data to the file determined by filepath + + @param filepath: string. eg. '/path/to/filename' + @param data: list of lines to write to filepath + """ + if not self.options["quiet"]: + print(' - Saving file: %s' %filepath) + with open(filepath, "w") as output: + output.write('\n'.join(data)) + print(" - Done") + + +def main(input_args): + """Common starting method by the analyze master + unless all modules are converted to this class method. + + @param input_args: input args as supplied by equery master module. + """ + query_module = Rebuild() + query_module.run(input_args, gentoolkit.CONFIG['quiet']) + +# vim: set ts=4 sw=4 tw=79: + -- cgit v1.2.3