diff options
author | Robin H. Johnson <robbat2@gentoo.org> | 2008-11-30 08:53:22 +0000 |
---|---|---|
committer | Robin H. Johnson <robbat2@gentoo.org> | 2008-11-30 08:53:22 +0000 |
commit | f9df9bbcd3aa084244563386d791cf3aba360419 (patch) | |
tree | 8f9a3741b380055d2c5f190d1b8d8f3d0c17fa50 | |
parent | Ignore junk. (diff) | |
download | packages-3-f9df9bbcd3aa084244563386d791cf3aba360419.tar.gz packages-3-f9df9bbcd3aa084244563386d791cf3aba360419.tar.bz2 packages-3-f9df9bbcd3aa084244563386d791cf3aba360419.zip |
Remove explicit copy of pkgcore.
202 files changed, 0 insertions, 29180 deletions
diff --git a/pkgcore/__init__.py b/pkgcore/__init__.py deleted file mode 100644 index a9adf28..0000000 --- a/pkgcore/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# Copyright: 2005 Brian Harring <ferringb@gmail.com> -# License: GPL2 - diff --git a/pkgcore/bin/ebuild-env/ebuild-daemon.lib b/pkgcore/bin/ebuild-env/ebuild-daemon.lib deleted file mode 100755 index 9bc1b91..0000000 --- a/pkgcore/bin/ebuild-env/ebuild-daemon.lib +++ /dev/null @@ -1,62 +0,0 @@ -#!/bin/bash -# ebuild-daemon.lib; daemon lib code. -# Copyright 2005-2006 Brian Harring <ferringb@gmail.com> - -alias die='diefunc "$FUNCNAME" "$LINENO" "$?"' -#alias listen='read -u 3 -t 10' -alias assert='_pipestatus="${PIPESTATUS[*]}"; [[ "${_pipestatus// /}" -eq 0 ]] || diefunc "$FUNCNAME" "$LINENO" "$_pipestatus"' - - -# ask the python side to display sandbox complaints. -request_sandbox_summary() { - local line - speak "request_sandbox_summary ${SANDBOX_LOG}" - listen line - while [ "$line" != "end_sandbox_summary" ]; do - echo "$line" - listen line - done -} - -internal_inherit() { - local line - if [ "$#" != "1" ]; then - die "internal_inherit accepts one arg, requested eclass location. $* is a bit much" - fi - speak "request_inherit $1" - listen line - if [ "$line" == "path" ]; then - listen line; - source "${line}" || die "failed sources inherit: ${line}" - elif [ "$line" == "transfer" ]; then - listen line; - eval "$line" || die "failed evaluating eclass $x on an inherit transfer" - elif [ "$line" == "failed" ]; then - die "inherit for $x failed" - else - die "unknown inherit command from pythonic side, '$line' for eclass $x" - fi -} - -source_profiles() { - local line - speak request_profiles - listen line - while [ "$line" != end_request ]; do - if [ "$line" == "path" ]; then - listen line; - source "${line}" - elif [ "$line" == "transfer" ]; then - listen line; - eval "$line" || die "failed evaluating profile bashrc: ${line}" - else - speak "failed" - die "unknown profile bashrc transfer mode from pythonic side, '$line'" - fi - speak "next" - listen line - done -} -DONT_EXPORT_FUNCS="${DONT_EXPORT_FUNCS} $(declare -F | cut -s -d ' ' -f 3)" - -: diff --git a/pkgcore/bin/ebuild-env/ebuild-daemon.sh b/pkgcore/bin/ebuild-env/ebuild-daemon.sh deleted file mode 100755 index f2ae35f..0000000 --- a/pkgcore/bin/ebuild-env/ebuild-daemon.sh +++ /dev/null @@ -1,266 +0,0 @@ -#!/bin/bash -# ebuild-daemon.sh; core ebuild processor handling code -# Copyright 2004-2006 Brian Harring <ferringb@gmail.com> - -alias die='diefunc "$FUNCNAME" "$LINENO" "$?"' -#alias listen='read -u 3 -t 10' -alias assert='_pipestatus="${PIPESTATUS[*]}"; [[ "${_pipestatus// /}" -eq 0 ]] || diefunc "$FUNCNAME" "$LINENO" "$_pipestatus"' - -# use listen/speak for talking to the running portage instance instead of echo'ing to the fd yourself. -# this allows us to move the open fd's w/out issues down the line. -listen() { - if ! read -u ${EBD_READ_FD} $1; then - echo "coms error, read failed: backing out of daemon." - exit 1 - fi -} - -speak() { - echo "$*" >&${EBD_WRITE_FD} -} -declare -rf speak -declare -r EBD_WRITE_FD EBD_READ_FD -# ensure the other side is still there. Well, this moreso is for the python side to ensure -# loading up the intermediate funcs succeeded. -listen com -if [ "$com" != "dude?" ]; then - echo "serv init coms failed, received $com when expecting 'dude?'" - exit 1 -fi -speak "dude!" -listen PKGCORE_BIN_PATH -[ -z "$PKGCORE_BIN_PATH" ] && die "PKGCORE_BIN_PATH=$PKGCORE_BIN_PATH , bailing" -declare -rx PKGCORE_BIN_PATH -listen PKGCORE_PYTHON -[ -z "$PKGCORE_PYTHON" ] && die "empty PKGCORE_PYTHON, bailing" -declare -rx PKGCORE_PYTHON -listen PKGCORE_PYTHONPATH -[ -z "$PKGCORE_PYTHONPATH" ] && die "empty PKGCORE_PYTHONPATH, bailing" -declare -rx PKGCORE_PYTHONPATH - -if ! source "${PKGCORE_BIN_PATH}/ebuild.sh" daemonize; then - speak "failed" - die "failed sourcing ${PKGCORE_BIN_PATH}/ebuild.sh" -fi - -if [ -n "$SANDBOX_LOG" ]; then - listen com - if [ "$com" != "sandbox_log?" ]; then - echo "unknown com '$com'" - exit 1 - fi - speak "$SANDBOX_LOG" - declare -rx SANDBOX_LOG="$SANDBOX_LOG" # #="/tmp/sandbox-${P}-${PORTAGE_SANDBOX_PID}.log" - addwrite $SANDBOX_LOG -fi - -alive='1' -re="$(readonly | cut -s -d '=' -f 1 | cut -s -d ' ' -f 3)" -for x in $re; do - if ! hasq $x "$DONT_EXPORT_VARS"; then - DONT_EXPORT_VARS="${DONT_EXPORT_VARS} $x" - fi -done -speak $re -unset x re - - -if ! source "${PKGCORE_BIN_PATH}/ebuild-daemon.lib"; then - speak failed - die "failed source ${PKGCORE_BIN_PATH}/ebuild-daemon.lib" -fi - -DONT_EXPORT_FUNCS="$(declare -F | cut -s -d ' ' -f 3)" -DONT_EXPORT_VARS="${DONT_EXPORT_VARS} alive com PORTAGE_LOGFILE cont" - -# depend's speed up. turn on qa interceptors by default, instead of flipping them on for each depends -# call. -export QA_CONTROLLED_EXTERNALLY="yes" -enable_qa_interceptors - -if ! source "${PKGCORE_BIN_PATH}/ebuild-functions.sh"; then - speak failed - die "failed sourcing ${PORTAGE_LIB}/ebuild-functions.sh" -fi - -export PORTAGE_PRELOADED_ECLASSES='' -unset_colors - - -sigint_handler() { - EBD_DISABLE_DIEFUNC="asdf" - exec 2>/dev/null - exec 1>/dev/null - kill -2 $PPID - speak "killed" - # this relies on the python side to *not* discard the killed - #exit 2 -} -trap sigint_handler SIGINT - -sigkill_handler() { - EBD_DISABLE_DIEFUNC="asdf" - exec 2>/dev/null - exec 1>/dev/null - kill -9 $$PID - speak "killed" - exit 9 -} - -trap sigkill_handler SIGKILL - -while [ "$alive" == "1" ]; do - com='' - listen com - case $com in - process_ebuild*) - # cleanse whitespace. - phases="$(echo ${com#process_ebuild})" - PORTAGE_SANDBOX_PID="$PPID" - # note the (; forks. prevents the initialized ebd env from being polluted by ebuild calls. - ( - if [ "${phases/depend/}" == "$phases" ]; then - disable_qa_interceptors - fi - line='' - cont=0 - - while [ "$cont" == 0 ]; do - line='' - listen line - if [ "$line" == "start_receiving_env" ]; then - while listen line && [ "$line" != "end_receiving_env" ]; do #[ "$line" != "end_receiving_env" ]; do - save_IFS - IFS=$'\0' - eval ${line}; - val=$?; - restore_IFS - if [ $val != "0" ]; then - echo "err, env receiving threw an error for '$line': $?" >&2 - speak "env_receiving_failed" - cont=1 - break - fi - if [ "${on:-unset}" != "unset" ]; then - echo "sudo = ${SUDO_COMMAND}" >&2 - declare | grep -i sudo_command >&@ - echo "disabling" >&2 - unset on - fi - done - if [ "$cont" == "0" ]; then - speak "env_received" - fi - elif [ "${line:0:7}" == "logging" ]; then - PORTAGE_LOGFILE="$(echo ${line#logging})" - speak "logging_ack" - elif [ "${line:0:17}" == "set_sandbox_state" ]; then - if [ $((${line:18})) -eq 0 ]; then - export SANDBOX_DISABLED=1 - else - export SANDBOX_DISABLED=0 - export SANDBOX_VERBOSE="no" - fi - elif [ "${line}" == "start_processing" ]; then - cont=2 - else - echo "received unknown com: $line" >&2 - fi - done - if [ "$cont" != 2 ]; then - exit $cont - else - reset_sandbox - if [ -n "$SANDBOX_LOG" ]; then - addwrite $SANDBOX_LOG - if [ -n "$PORTAGE_LOGFILE" ]; then - addwrite "$PORTAGE_LOGFILE" - fi - fi - if [ -z $RC_NOCOLOR ]; then - set_colors - fi - - DONT_EXPORT_FUNCS="${DONT_EXPORT_FUNCS} ${PORTAGE_PRELOADED_ECLASSES}" - for x in $DONT_EXPORT_FUNCS; do - declare -fr $x &> /dev/null - done - for e in $phases; do - umask 0022 - if [ -z $PORTAGE_LOGFILE ]; then - execute_phases ${e} - ret=$? - else - # why do it this way rather then the old '[ -f ${T}/.succesfull }'? - # simple. this allows the actual exit code to be used, rather then just stating no .success == 1 || 0 - # note this was - # execute_phases ${e] &> >(umask 0002; tee -i -a $PORTAGE_LOGFILE) - # less then bash v3 however hates it. And I hate less then v3. - # circle of hate you see. - execute_phases ${e} 2>&1 | { - # this applies to the subshell only. - umask 0002 - tee -i -a $PORTAGE_LOGFILE - } - ret=${PIPESTATUS[0]} - fi - # if sandbox log exists, then there were complaints from it. - # tell python to display the errors, then dump relevant vars for debugging. - if [ -n "$SANDBOX_LOG" ] && [ -e "$SANDBOX_LOG" ]; then - ret=1 - echo "sandbox exists- $SANDBOX_LOG" - request_sandbox_summary - echo "SANDBOX_ON:=${SANDBOX_ON:-unset}" >&2 - echo "SANDBOX_DISABLED:=${SANDBOX_DISABLED:-unset}" >&2 - echo "SANDBOX_READ:=${SANDBOX_READ:-unset}" >&2 - echo "SANDBOX_WRITE:=${SANDBOX_WRITE:-unset}" >&2 - echo "SANDBOX_PREDICT:=${SANDBOX_PREDICT:-unset}" >&2 - echo "SANDBOX_DEBUG:=${SANDBOX_DEBUG:-unset}" >&2 - echo "SANDBOX_DEBUG_LOG:=${SANDBOX_DEBUG_LOG:-unset}" >&2 - echo "SANDBOX_LOG:=${SANDBOX_LOG:-unset}" >&2 - echo "SANDBOX_ARMED:=${SANDBOX_ARMED:-unset}" >&2 - fi - if [ "$ret" != "0" ]; then - exit $(($ret)) - fi - done - fi - ) - # post fork. tell python if it succeeded or not. - if [ $? != 0 ]; then - echo "phases failed" - speak "phases failed" - else - speak "phases succeeded" - fi - ;; - shutdown_daemon) - alive="0" - ;; - preload_eclass*) - echo "preloading eclasses into funcs." >&2 - disable_qa_interceptors - success="succeeded" - com="${com#preload_eclass }" - for e in ${com}; do - x="${e##*/}" - x="${x%.eclass}" - echo "preloading eclass $x" >&2 - if ! bash -n "$e"; then - echo "errors detected in '$e'" >&2 - success='failed' - break - fi - y="$( < $e)" - eval "eclass_${x}_inherit() { - $y - }" - done - speak "preload_eclass ${success}" - unset e x y success - enable_qa_interceptors - export PORTAGE_PRELOADED_ECLASSES="$PORTAGE_PRELOADED_ECLASSES ${com}" - ;; - esac -done -exit 0 diff --git a/pkgcore/bin/ebuild-env/ebuild-default-functions.sh b/pkgcore/bin/ebuild-env/ebuild-default-functions.sh deleted file mode 100755 index fc299bc..0000000 --- a/pkgcore/bin/ebuild-env/ebuild-default-functions.sh +++ /dev/null @@ -1,903 +0,0 @@ -#!/bin/bash -# ebuild-default-functions.sh; default functions for ebuild env that aren't saved- specific to the portage instance. -# Copyright 2005-2006 Brian Harring <ferringb@gmail.com> -# Copyright 2004-2006 Gentoo Foundation - -portageq() { - if [[ $EBUILD_PHASE == depend ]]; then - die "portageq calls in depends phase is disallowed" - fi - PYTHONPATH="$PKGCORE_PYTHONPATH" \ - "${PKGCORE_PYTHON}" "${PKGCORE_BIN_PATH}/portageq_emulation" \ - --domain "${PKGCORE_DOMAIN}" "$@" -} - -has_version() -{ - portageq 'has_version' "${ROOT}" "$1" -} - -best_version() -{ - portageq 'best_version' "${ROOT}" "$1" -} - -check_KV() -{ - if [ -z "${KV}" ]; then - eerror "" - eerror "Could not determine your kernel version." - eerror "Make sure that you have /usr/src/linux symlink." - eerror "And that said kernel has been configured." - eerror "You can also simply run the following command" - eerror "in the kernel referenced by /usr/src/linux:" - eerror " make include/linux/version.h" - eerror "" - die - fi -} - -# adds ".keep" files so that dirs aren't auto-cleaned -keepdir() -{ - dodir "$@" - local x - if [ "$1" == "-R" ] || [ "$1" == "-r" ]; then - shift - find "$@" -type d -printf "${D}/%p/.keep\n" | tr "\n" "\0" | $XARGS -0 -n100 touch || die "Failed to recursive create .keep files" - else - for x in "$@"; do - touch "${D}/${x}/.keep" || die "Failed to create .keep in ${D}/${x}" - done - fi -} - -# sandbox support functions -addread() -{ - export SANDBOX_READ="$SANDBOX_READ:$1" -} - -addwrite() -{ - export SANDBOX_WRITE="$SANDBOX_WRITE:$1" -} - -adddeny() -{ - export SANDBOX_DENY="$SANDBOX_DENY:$1" -} - -addpredict() -{ - export SANDBOX_PREDICT="$SANDBOX_PREDICT:$1" -} - -unpack() -{ - local x y myfail srcdir taropts tar_subdir - taropts='--no-same-owner' - - [ -z "$*" ] && die "Nothing passed to the 'unpack' command" - - for x in "$@"; do - echo ">>> Unpacking ${x} to ${PWD}" - myfail="failure unpacking ${x}" - y="${x%.*}" - y="${y##*.}" - if [ "${x:0:2}" == "./" ]; then - srcdir='' - else - srcdir="${DISTDIR}" - fi - - [ ! -s "${srcdir}${x}" ] && die "$myfail: empty file" - [ "${x/${DISTDIR}}" != "${x}" ] && \ - die "Arguments to unpack() should not begin with \${DISTDIR}." - - case "${x}" in - *.tar) - tar xf "${srcdir}${x}" ${taropts} || die "$myfail" - ;; - *.tar.gz|*.tgz|*.tar.Z) - tar xzf "${srcdir}${x}" ${taropts} || die "$myfail" - ;; - *.tar.bz2|*.tbz2|*.tbz) - bzip2 -dc "${srcdir}${x}" | tar xf - ${taropts} - assert "$myfail" - ;; - *.ZIP|*.zip|*.jar) - unzip -qo "${srcdir}${x}" || die "$myfail" - ;; - *.gz|*.Z|*.z) - gzip -dc "${srcdir}${x}" > ${x%.*} || die "$myfail" - ;; - *.bz2|*.bz) - bzip2 -dc "${srcdir}${x}" > ${x%.*} || die "$myfail" - ;; - *.7Z|*.7z) - local my_output - my_output="$(7z x -y "${srcdir}/${x}")" - if [ $? -ne 0 ]; then - echo "${my_output}" >&2 - die "$myfail" - fi - ;; - *.RAR|*.rar) - unrar x -idq -o+ "${srcdir}/${x}" || die "$myfail" - ;; - *.LHa|*.LHA|*.lha|*.lzh) - lha xfq "${srcdir}/${x}" || die "$myfail" - ;; - *.a|*.deb) - ar x "${srcdir}/${x}" || die "$myfail" - ;; - *) - echo "unpack ${x}: file format not recognized. Ignoring." - ;; - esac - done - find . -mindepth 1 -maxdepth 1 ! -type l -print0 | \ - ${XARGS} -0 chmod -fR a+rX,u+w,g-w,o-w - -} - -dyn_setup() -{ - MUST_EXPORT_ENV="yes" - pkg_setup -} - -dyn_unpack() -{ - local newstuff="no" - MUST_EXPORT_ENV="yes" - if [ -e "${WORKDIR}" ]; then - local x - local checkme - for x in ${AA}; do - echo ">>> Checking ${x}'s mtime..." - if [ "${DISTDIR}/${x}" -nt "${WORKDIR}" ]; then - echo ">>> ${x} has been updated; recreating WORKDIR..." - newstuff="yes" - rm -rf "${WORKDIR}" - break - fi - done - if [ "${EBUILD}" -nt "${WORKDIR}" ]; then - echo ">>> ${EBUILD} has been updated; recreating WORKDIR..." - newstuff="yes" - rm -rf "${WORKDIR}" - fi - fi - - cd "${WORKDIR}" - src_unpack -} - -abort_handler() -{ - local msg - if [ "$2" != "fail" ]; then - msg="${EBUILD}: ${1} aborted; exiting." - else - msg="${EBUILD}: ${1} failed; exiting." - fi - echo - echo "$msg" - echo - eval ${3} - #unset signal handler -} - -abort_compile() -{ - abort_handler "src_compile" $1 - exit 1 -} - -abort_unpack() -{ - abort_handler "src_unpack" $1 - exit 1 -} - -abort_package() -{ - abort_handler "dyn_package" $1 - rm -f "${PKGDIR}"/All/${PF}.t* - exit 1 -} - -abort_test() -{ - abort_handler "dyn_test" $1 - exit 1 -} - -abort_install() -{ - abort_handler "src_install" $1 - exit 1 -} - -dyn_compile() -{ - MUST_EXPORT_ENV="yes" - export DESTTREE=/usr - export INSDESTTREE="" - export EXEDESTTREE="" - export DOCDESTTREE="" - export INSOPTIONS="-m0644" - export EXEOPTIONS="-m0755" - export LIBOPTIONS="-m0644" - export DIROPTIONS="-m0755" - export MOPREFIX=${PN} - - [ "${CFLAGS-unset}" != "unset" ] && export CFLAGS - [ "${CXXFLAGS-unset}" != "unset" ] && export CXXFLAGS - [ "${LIBCFLAGS-unset}" != "unset" ] && export LIBCFLAGS - [ "${LIBCXXFLAGS-unset}" != "unset" ] && export LIBCXXFLAGS - [ "${LDFLAGS-unset}" != "unset" ] && export LDFLAGS - [ "${ASFLAGS-unset}" != "unset" ] && export ASFLAGS - - [ ! -z "${DISTCC_DIR}" ] && addwrite "${DISTCC_DIR}" - - if [ -d "${S}" ]; then - cd "${S}" - else - # cd to some random dir that we at least control. - cd "${WORKDIR}" - fi - #our custom version of libtool uses $S and $D to fix - #invalid paths in .la files - export S D - #some packages use an alternative to $S to build in, cause - #our libtool to create problematic .la files - export PWORKDIR="$WORKDIR" - src_compile - #|| abort_compile "fail" - if hasq nostrip $FEATURES $RESTRICT; then - touch DEBUGBUILD - fi -} - - -dyn_test() -{ - echo ">>> Test phase [enabled]: ${CATEGORY}/${PF}" - MUST_EXPORT_ENV="yes" - if [ -d "${S}" ]; then - cd "${S}" - else - cd "${WORKDIR}" - fi - src_test -} - - -dyn_install() -{ - rm -rf "${D}" - mkdir "${D}" - if [ -d "${S}" ]; then - cd "${S}" - else - cd "$WORKDIR" - fi - echo - echo ">>> Install ${PF} into ${D} category ${CATEGORY}" - #our custom version of libtool uses $S and $D to fix - #invalid paths in .la files - export S D - #some packages uses an alternative to $S to build in, cause - #our libtool to create problematic .la files - export PWORKDIR="$WORKDIR" - src_install - #|| abort_install "fail" - prepall - cd "${D}" - - if type -p scanelf > /dev/null ; then - # Make sure we disallow insecure RUNPATH/RPATH's - # Don't want paths that point to the tree where the package was built - # (older, broken libtools would do this). Also check for null paths - # because the loader will search $PWD when it finds null paths. - f=$(scanelf -qyRF '%r %p' "${D}" | grep -E "(${WORKDIR}|${D}|: |::|^ )") - if [[ -n ${f} ]] ; then - echo -ne '\a\n' - echo "QA Notice: the following files contain insecure RUNPATH's" - echo " Please file a bug about this at http://bugs.gentoo.org/" - echo " For more information on this issue, kindly review:" - echo " http://bugs.gentoo.org/81745" - echo "${f}" - echo -ne '\a\n' - die "Insecure binaries detected" - fi - - # Check for setid binaries but are not built with BIND_NOW - f=$(scanelf -qyRF '%b %p' "${D}") - if [[ -n ${f} ]] ; then - echo -ne '\a\n' - echo "QA Notice: the following files are setXid, dyn linked, and using lazy bindings" - echo " This combination is generally discouraged. Try re-emerging the package:" - echo " LDFLAGS='-Wl,-z,now' emerge ${PN}" - echo "${f}" - echo -ne '\a\n' - [[ ${FEATURES/stricter} != "${FEATURES}" ]] \ - && die "Aborting due to lazy bindings" - sleep 1 - fi - - # TEXTREL's are baaaaaaaad - f=$(scanelf -qyRF '%t %p' "${D}") - if [[ -n ${f} ]] ; then - echo -ne '\a\n' - echo "QA Notice: the following files contain runtime text relocations" - echo " Text relocations require a lot of extra work to be preformed by the" - echo " dynamic linker which will cause serious performance impact on IA-32" - echo " and might not function properly on other architectures hppa for example." - echo " If you are a programmer please take a closer look at this package and" - echo " consider writing a patch which addresses this problem." - echo "${f}" - echo -ne '\a\n' - [[ ${FEATURES/stricter} != "${FEATURES}" ]] \ - && die "Aborting due to textrels" - sleep 1 - fi - - # Check for files with executable stacks - f=$(scanelf -qyRF '%e %p' "${D}") - if [[ -n ${f} ]] ; then - echo -ne '\a\n' - echo "QA Notice: the following files contain executable stacks" - echo " Files with executable stacks will not work properly (or at all!)" - echo " on some architectures/operating systems. A bug should be filed" - echo " at http://bugs.gentoo.org/ to make sure the file is fixed." - echo "${f}" - echo -ne '\a\n' - [[ ${FEATURES/stricter} != "${FEATURES}" ]] \ - && die "Aborting due to +x stack" - sleep 1 - fi - - # disabled by harring; we don't use it currently. - # Save NEEDED information - #scanelf -qyRF '%p %n' "${D}" | sed -e 's:^:/:' > "${T}/NEEDED" - fi - - if hasq multilib-strict ${FEATURES} && [ -x /usr/bin/file -a -x /usr/bin/find -a \ - -n "${MULTILIB_STRICT_DIRS}" -a -n "${MULTILIB_STRICT_DENY}" ]; then - MULTILIB_STRICT_EXEMPT=${MULTILIB_STRICT_EXEMPT:-"(perl5|gcc|gcc-lib)"} - for dir in ${MULTILIB_STRICT_DIRS}; do - [ -d "${D}/${dir}" ] || continue - for file in $(find ${D}/${dir} -type f | egrep -v "^${D}/${dir}/${MULTILIB_STRICT_EXEMPT}"); do - file ${file} | egrep -q "${MULTILIB_STRICT_DENY}" && die "File ${file} matches a file type that is not allowed in ${dir}" - done - done - fi - - echo ">>> Completed installing ${PF} into ${D}" - echo - unset dir - MUST_EXPORT_ENV="yes" -} - -dyn_postinst() -{ - pkg_postinst -} - -dyn_preinst() -{ - # set IMAGE depending if this is a binary or compile merge - local IMAGE=${D} - - # Make sure D is where the package expects it - D=${IMAGE} pkg_preinst - - # Smart FileSystem Permissions - if hasq sfperms $FEATURES; then - for i in $(find "${IMAGE}"/ -type f -perm -4000); do - ebegin ">>> SetUID: [chmod go-r] $i " - chmod go-r "$i" - eend $? - done - for i in $(find "${IMAGE}"/ -type f -perm -2000); do - ebegin ">>> SetGID: [chmod o-r] $i " - chmod o-r "$i" - eend $? - done - fi - - # total suid control. - if hasq suidctl $FEATURES > /dev/null ; then - sfconf=/etc/portage/suidctl.conf - echo ">>> Preforming suid scan in ${IMAGE}" - for i in $(find "${IMAGE}"/ -type f \( -perm -4000 -o -perm -2000 \) ); do - if [ -s "${sfconf}" ]; then - suid=$(grep ^${i/${IMAGE}/}$ ${sfconf}) - if [ "${suid}" = "${i/${IMAGE}/}" ]; then - echo "- ${i/${IMAGE}/} is an approved suid file" - else - echo ">>> Removing sbit on non registered ${i/${IMAGE}/}" - sleepbeep 6 - chmod ugo-s "${i}" - grep ^#${i/${IMAGE}/}$ ${sfconf} > /dev/null || { - # sandbox prevents us from writing directly - # to files outside of the sandbox, but this - # can easly be bypassed using the addwrite() function - addwrite "${sfconf}" - echo ">>> Appending commented out entry to ${sfconf} for ${PF}" - ls_ret=`ls -ldh "${i}"` - echo "## ${ls_ret%${IMAGE}*}${ls_ret#*${IMAGE}}" >> ${sfconf} - echo "#${i/${IMAGE}/}" >> ${sfconf} - # no delwrite() eh? - # delwrite ${sconf} - } - fi - else - echo "suidctl feature set but you are lacking a ${sfconf}" - fi - done - fi - - # SELinux file labeling (needs to always be last in dyn_preinst) - if hasq selinux $FEATURES || use selinux; then - # only attempt to label if setfiles is executable - # and 'context' is available on selinuxfs. - if [ -f /selinux/context -a -x /usr/sbin/setfiles ]; then - echo ">>> Setting SELinux security labels" - if [ -f ${POLICYDIR}/file_contexts/file_contexts ]; then - cp -f "${POLICYDIR}/file_contexts/file_contexts" "${T}" - else - make -C "${POLICYDIR}" FC=${T}/file_contexts "${T}/file_contexts" - fi - - addwrite /selinux/context - /usr/sbin/setfiles -r "${IMAGE}" "${T}/file_contexts" "${IMAGE}" \ - || die "Failed to set SELinux security labels." - else - # nonfatal, since merging can happen outside a SE kernel - # like during a recovery situation - echo "!!! Unable to set SELinux security labels" - fi - fi - MUST_EXPORT_ENV="yes" -} - - -# debug-print() gets called from many places with verbose status information useful -# for tracking down problems. The output is in $T/eclass-debug.log. -# You can set ECLASS_DEBUG_OUTPUT to redirect the output somewhere else as well. -# The special "on" setting echoes the information, mixing it with the rest of the -# emerge output. -# You can override the setting by exporting a new one from the console, or you can -# set a new default in make.*. Here the default is "" or unset. - -# in the future might use e* from /etc/init.d/functions.sh if i feel like it -debug-print() -{ - if [ "$EBUILD_PHASE" == "depend" ] && [ -z "${PKGCORE_DEBUG}" ]; then - return - fi - # if $T isn't defined, we're in dep calculation mode and - # shouldn't do anything - [ -z "$T" ] && return 0 - - while [ "$1" ]; do - - # extra user-configurable targets - if [ "$ECLASS_DEBUG_OUTPUT" == "on" ]; then - echo "debug: $1" - elif [ -n "$ECLASS_DEBUG_OUTPUT" ]; then - echo "debug: $1" >> $ECLASS_DEBUG_OUTPUT - fi - - # default target - echo "$1" >> "${T}/eclass-debug.log" - # let the portage user own/write to this file - chmod g+w "${T}/eclass-debug.log" &>/dev/null - - shift - done -} - -# The following 2 functions are debug-print() wrappers - -debug-print-function() -{ - str="$1: entering function" - shift - debug-print "$str, parameters: $*" -} - -debug-print-section() -{ - debug-print "now in section $*" -} - - -internal_inherit() -{ - # default, backwards compatible beast. - local location overlay - location="${ECLASSDIR}/${1}.eclass" - - if [ -n "$PORTDIR_OVERLAY" ]; then - local overlay - for overlay in ${PORTDIR_OVERLAY}; do - if [ -e "${overlay}/eclass/${1}.eclass" ]; then - location="${overlay}/eclass/${1}.eclass" - debug-print " eclass exists: ${location}" - fi - done - fi - debug-print "inherit: $1 -> $location" - source "$location" || die "died sourcing $location in inherit()" - return 0 -} - -# Sources all eclasses in parameters -declare -ix ECLASS_DEPTH=0 -inherit() -{ - local SAVED_INHERIT_COUNT=0 INHERITED_ALREADY=0 - - if [[ $ECLASS_DEPTH < 0 ]] && [ "${EBUILD_PHASE}" == "depend" ]; then - echo "QA Notice: ${CATEGORY}/${PF} makes multiple inherit calls: $1" >&2 - SAVED_INHERIT_COUNT=$ECLASS_DEPTH - ECLASS_DEPTH=0 - fi - if hasq $1 $INHERITED && [ "${EBUILD_PHASE}" == "depend" ]; then - #echo "QA notice: $1 is inherited multiple times: ${CATEGORY}/${PF}" >&2 - INHERITED_ALREADY=1 - fi - ECLASS_DEPTH=$(($ECLASS_DEPTH + 1)) - if [[ $ECLASS_DEPTH > 1 ]]; then - debug-print "*** Multiple Inheritence (Level: ${ECLASS_DEPTH})" - fi - - local location olocation - local PECLASS - - local B_IUSE - local B_DEPEND - local B_RDEPEND - local B_CDEPEND - local B_PDEPEND - while [ -n "$1" ]; do - - # PECLASS is used to restore the ECLASS var after recursion. - PECLASS="$ECLASS" - export ECLASS="$1" - - if [ "$EBUILD_PHASE" != "depend" ]; then - if ! hasq $ECLASS $INHERITED; then - echo - echo "QA Notice: ECLASS '$ECLASS' illegal conditional inherit in $CATEGORY/$PF" >&2 - echo - fi - fi - - #We need to back up the value of DEPEND and RDEPEND to B_DEPEND and B_RDEPEND - #(if set).. and then restore them after the inherit call. - - #turn off glob expansion - set -f - - # Retain the old data and restore it later. - unset B_IUSE B_DEPEND B_RDEPEND B_CDEPEND B_PDEPEND - [ "${IUSE-unset}" != "unset" ] && B_IUSE="${IUSE}" - [ "${DEPEND-unset}" != "unset" ] && B_DEPEND="${DEPEND}" - [ "${RDEPEND-unset}" != "unset" ] && B_RDEPEND="${RDEPEND}" - [ "${CDEPEND-unset}" != "unset" ] && B_CDEPEND="${CDEPEND}" - [ "${PDEPEND-unset}" != "unset" ] && B_PDEPEND="${PDEPEND}" - unset IUSE DEPEND RDEPEND CDEPEND PDEPEND - #turn on glob expansion - set +f - if ! internal_inherit "$1"; then - die "failed sourcing $1 in inherit()" - fi - - #turn off glob expansion - set -f - - # If each var has a value, append it to the global variable E_* to - # be applied after everything is finished. New incremental behavior. - [ "${IUSE-unset}" != "unset" ] && export E_IUSE="${E_IUSE} ${IUSE}" - [ "${DEPEND-unset}" != "unset" ] && export E_DEPEND="${E_DEPEND} ${DEPEND}" - [ "${RDEPEND-unset}" != "unset" ] && export E_RDEPEND="${E_RDEPEND} ${RDEPEND}" - [ "${CDEPEND-unset}" != "unset" ] && export E_CDEPEND="${E_CDEPEND} ${CDEPEND}" - [ "${PDEPEND-unset}" != "unset" ] && export E_PDEPEND="${E_PDEPEND} ${PDEPEND}" - - [ "${B_IUSE-unset}" != "unset" ] && IUSE="${B_IUSE}" - [ "${B_IUSE-unset}" != "unset" ] || unset IUSE - - [ "${B_DEPEND-unset}" != "unset" ] && DEPEND="${B_DEPEND}" - [ "${B_DEPEND-unset}" != "unset" ] || unset DEPEND - - [ "${B_RDEPEND-unset}" != "unset" ] && RDEPEND="${B_RDEPEND}" - [ "${B_RDEPEND-unset}" != "unset" ] || unset RDEPEND - - [ "${B_CDEPEND-unset}" != "unset" ] && CDEPEND="${B_CDEPEND}" - [ "${B_CDEPEND-unset}" != "unset" ] || unset CDEPEND - - [ "${B_PDEPEND-unset}" != "unset" ] && PDEPEND="${B_PDEPEND}" - [ "${B_PDEPEND-unset}" != "unset" ] || unset PDEPEND - - #turn on glob expansion - set +f - - if hasq $1 $INHERITED && [ $INHERITED_ALREADY == 0 ]; then -# -# enable this one eclasses no longer fool with eclass and inherited. -# if [ "${EBUILD_PHASE}" == "depend" ]; then -# echo "QA Notice: ${CATEGORY}/${PF}: eclass $1 is incorrectly setting \$INHERITED." >&2 -# fi - : - else - INHERITED="$INHERITED $ECLASS" - fi - export ECLASS="$PECLASS" - - shift - done - ECLASS_DEPTH=$(($ECLASS_DEPTH - 1)) - if [[ $ECLASS_DEPTH == 0 ]]; then - ECLASS_DEPTH=$(($SAVED_INHERIT_COUNT - 1)) - fi -} - -# Exports stub functions that call the eclass's functions, thereby making them default. -# For example, if ECLASS="base" and you call "EXPORT_FUNCTIONS src_unpack", the following -# code will be eval'd: -# src_unpack() { base_src_unpack; } -EXPORT_FUNCTIONS() -{ - if [ -z "$ECLASS" ]; then - echo "EXPORT_FUNCTIONS without a defined ECLASS" >&2 - exit 1 - fi - while [ "$1" ]; do - debug-print "EXPORT_FUNCTIONS: ${1} -> ${ECLASS}_${1}" - eval "$1() { ${ECLASS}_$1 "\$@" ; }" > /dev/null - shift - done -} - -# adds all parameters to E_DEPEND and E_RDEPEND, which get added to DEPEND -# and RDEPEND after the ebuild has been processed. This is important to -# allow users to use DEPEND="foo" without frying dependencies added by an -# earlier inherit. It also allows RDEPEND to work properly, since a lot -# of ebuilds assume that an unset RDEPEND gets its value from DEPEND. -# Without eclasses, this is true. But with them, the eclass may set -# RDEPEND itself (or at least used to) which would prevent RDEPEND from -# getting its value from DEPEND. This is a side-effect that made eclasses -# have unreliable dependencies. - -newdepend() -{ - debug-print-function newdepend $* - debug-print "newdepend: E_DEPEND=$E_DEPEND E_RDEPEND=$E_RDEPEND" - - while [ -n "$1" ]; do - case $1 in - "/autotools") - do_newdepend DEPEND sys-devel/autoconf sys-devel/automake sys-devel/make - ;; - "/c") - do_newdepend DEPEND sys-devel/gcc virtual/libc - do_newdepend RDEPEND virtual/libc - ;; - *) - do_newdepend DEPEND $1 - ;; - esac - shift - done -} - -newrdepend() -{ - debug-print-function newrdepend $* - do_newdepend RDEPEND $1 -} - -newcdepend() -{ - debug-print-function newcdepend $* - do_newdepend CDEPEND $1 -} - -newpdepend() -{ - debug-print-function newpdepend $* - do_newdepend PDEPEND $1 -} - -do_newdepend() -{ - # This function does a generic change determining whether we're in an - # eclass or not. If we are, we change the E_* variables for deps. - debug-print-function do_newdepend $* - [ -z "$1" ] && die "do_newdepend without arguments" - - # Grab what we're affecting... Figure out if we're affecting eclasses. - [[ ${ECLASS_DEPTH} > 0 ]] && TARGET="E_$1" - [[ ${ECLASS_DEPTH} > 0 ]] || TARGET="$1" - shift # $1 was a variable name. - - while [ -n "$1" ]; do - # This bit of evil takes TARGET and uses it to evaluate down to a - # variable. This is a sneaky way to make this infinately expandable. - # The normal translation of this would look something like this: - # E_DEPEND="${E_DEPEND} $1" :::::: Cool, huh? :) - eval export ${TARGET}=\"\${${TARGET}} \$1\" - shift - done -} - -# this is a function for removing any directory matching a passed in pattern from -# PATH -remove_path_entry() -{ - save_IFS - IFS=":" - stripped_path="${PATH}" - while [ -n "$1" ]; do - cur_path="" - for p in ${stripped_path}; do - if [ "${p/${1}}" == "${p}" ]; then - cur_path="${cur_path}:${p}" - fi - done - stripped_path="${cur_path#:*}" - shift - done - restore_IFS - PATH="${stripped_path}" -} - -QA_INTERCEPTORS="javac java-config python python-config perl grep egrep fgrep sed gcc g++ cc bash awk nawk pkg-config" -enable_qa_interceptors() -{ - - # Turn of extended glob matching so that g++ doesn't get incorrectly matched. - shopt -u extglob - - # QA INTERCEPTORS - local FUNC_SRC BIN BODY BIN_PATH - for BIN in ${QA_INTERCEPTORS}; do - BIN_PATH=$(type -pf ${BIN}) - if [ "$?" != "0" ]; then - BODY="echo \"*** missing command: ${BIN}\" >&2; return 127" - else - BODY="${BIN_PATH} \"\$@\"; return \$?" - fi - FUNC_SRC="function ${BIN}() { - echo -n \"QA Notice: ${BIN} in global scope: \" >&2 - if [ \$ECLASS_DEPTH -gt 0 ]; then - echo \"eclass \${ECLASS}\" >&2 - else - echo \"\${CATEGORY}/\${PF}\" >&2 - fi - ${BODY} - }"; - eval "$FUNC_SRC" || echo "error creating QA interceptor ${BIN}" >&2 - done -} - -disable_qa_interceptors() -{ - for x in $QA_INTERCEPTORS; do - unset -f $x - done -} - -useq() -{ - local u="${1}" - local neg=0 - if [ "${u:0:1}" == "!" ]; then - u="${u:1}" - neg=1 - fi - local x - - # Make sure we have this USE flag in IUSE - # temp disable due to PORTAGE_ARCHLIST not being exported in - #if ! hasq "${u}" ${IUSE} ${E_IUSE} && ! hasq "${u}" ${PORTAGE_ARCHLIST} selinux; then - # echo "QA Notice: USE Flag '${u}' not in IUSE for ${CATEGORY}/${PF}" >&2 - #fi - - for x in ${USE}; do - if [ "${x}" == "${u}" ]; then - if [ ${neg} -eq 1 ]; then - return 1 - else - return 0 - fi - fi - done - if [ ${neg} -eq 1 ]; then - return 0 - else - return 1 - fi -} - -usev() -{ - if useq ${1}; then - echo "${1}" - return 0 - fi - return 1 -} - -# Used to generate the /lib/cpp and /usr/bin/cc wrappers -gen_wrapper() -{ - cat > $1 << END -#!/bin/sh - -$2 "\$@" -END - - chmod 0755 $1 -} - -insopts() -{ - INSOPTIONS="" - for x in $*; do - #if we have a debug build, let's not strip anything - if hasq nostrip $FEATURES $RESTRICT && [ "$x" == "-s" ]; then - continue - else - INSOPTIONS="$INSOPTIONS $x" - fi - done - export INSOPTIONS -} - -diropts() -{ - DIROPTIONS="" - for x in $*; do - DIROPTIONS="${DIROPTIONS} $x" - done - export DIROPTIONS -} - -exeopts() -{ - EXEOPTIONS="" - for x in $*; do - #if we have a debug build, let's not strip anything - if hasq nostrip $FEATURES $RESTRICT && [ "$x" == "-s" ]; then - continue - else - EXEOPTIONS="$EXEOPTIONS $x" - fi - done - export EXEOPTIONS -} - -libopts() -{ - LIBOPTIONS="" - for x in $*; do - #if we have a debug build, let's not strip anything - if hasq nostrip $FEATURES $RESTRICT && [ "$x" == "-s" ]; then - continue - else - LIBOPTIONS="$LIBOPTIONS $x" - fi - done - export LIBOPTIONS -} - -DONT_EXPORT_VARS="${DONT_EXPORT_VARS} ECLASS_DEPTH" -true diff --git a/pkgcore/bin/ebuild-env/ebuild-functions.sh b/pkgcore/bin/ebuild-env/ebuild-functions.sh deleted file mode 100755 index da60995..0000000 --- a/pkgcore/bin/ebuild-env/ebuild-functions.sh +++ /dev/null @@ -1,339 +0,0 @@ -#!/bin/bash -# ebuild-functions.sh; ebuild env functions, saved with the ebuild (not specific to the portage version). -# Copyright 2004-2005 Gentoo Foundation - -use() -{ - if useq ${1}; then - return 0 - fi - return 1 -} - -has() -{ - if hasq "$@"; then - return 0 - fi - return 1 -} - -use_with() -{ - if [ -z "$1" ]; then - echo "!!! use_with() called without a parameter." >&2 - echo "!!! use_with <USEFLAG> [<flagname> [value]]" >&2 - return - fi - - local UW_SUFFIX="" - if [ ! -z "${3}" ]; then - UW_SUFFIX="=${3}" - fi - - local UWORD="$2" - if [ -z "${UWORD}" ]; then - UWORD="$1" - fi - - if useq $1; then - echo "--with-${UWORD}${UW_SUFFIX}" - return 0 - else - echo "--without-${UWORD}" - return 1 - fi -} - -use_enable() -{ - if [ -z "$1" ]; then - echo "!!! use_enable() called without a parameter." >&2 - echo "!!! use_enable <USEFLAG> [<flagname> [value]]" >&2 - return - fi - - local UE_SUFFIX="" - if [ ! -z "${3}" ]; then - UE_SUFFIX="=${3}" - fi - - local UWORD="$2" - if [ -z "${UWORD}" ]; then - UWORD="$1" - fi - - if useq $1; then - echo "--enable-${UWORD}${UE_SUFFIX}" - return 0 - else - echo "--disable-${UWORD}" - return 1 - fi -} - -econf() -{ - local ret - ECONF_SOURCE="${ECONF_SOURCE:-.}" - if [ ! -x "${ECONF_SOURCE}/configure" ]; then - [ -f "${ECONF_SOURCE}/configure" ] && die "configure script isn't executable" - die "no configure script found" - fi - if ! hasq autoconfig $RESTRICT; then - if [ -e /usr/share/gnuconfig/ ]; then - local x - for x in $(find ${WORKDIR} -type f '(' -name config.guess -o -name config.sub ')' ); do - echo " * econf: updating ${x/${WORKDIR}\/} with /usr/share/gnuconfig/${x##*/}" - cp -f "/usr/share/gnuconfig/${x##*/}" "${x}" - chmod a+x "${x}" - done - fi - fi - if [ ! -z "${CBUILD}" ]; then - EXTRA_ECONF="--build=${CBUILD} ${EXTRA_ECONF}" - fi - - # if the profile defines a location to install libs to aside from default, pass it on. - # if the ebuild passes in --libdir, they're responsible for the conf_libdir fun. - LIBDIR_VAR="LIBDIR_${ABI}" - if [ -n "${ABI}" -a -n "${!LIBDIR_VAR}" ]; then - CONF_LIBDIR="${!LIBDIR_VAR}" - fi - unset LIBDIR_VAR - if [ -n "${CONF_LIBDIR}" ] && [ "${*/--libdir}" == "$*" ]; then - if [ "${*/--exec-prefix}" != "$*" ]; then - local args="$(echo $*)" - local -a pref=($(echo ${args/*--exec-prefix[= ]})) - CONF_PREFIX=${pref} - [ "${CONF_PREFIX:0:1}" != "/" ] && CONF_PREFIX="/${CONF_PREFIX}" - elif [ "${*/--prefix}" != "$*" ]; then - local args="$(echo $*)" - local -a pref=($(echo ${args/*--prefix[= ]})) - CONF_PREFIX=${pref} - [ "${CONF_PREFIX:0:1}" != "/" ] && CONF_PREFIX="/${CONF_PREFIX}" - else - CONF_PREFIX="/usr" - fi - export CONF_PREFIX - [ "${CONF_LIBDIR:0:1}" != "/" ] && CONF_LIBDIR="/${CONF_LIBDIR}" - - CONF_LIBDIR_RESULT="${CONF_PREFIX}${CONF_LIBDIR}" - for X in 1 2 3; do - # The escaping is weird. It will break if you escape the last one. - CONF_LIBDIR_RESULT="${CONF_LIBDIR_RESULT//\/\///}" - done - - EXTRA_ECONF="--libdir=${CONF_LIBDIR_RESULT} ${EXTRA_ECONF}" - fi - local EECONF_CACHE - echo ${ECONF_SOURCE}/configure \ - --prefix=/usr \ - --host=${CHOST} \ - --mandir=/usr/share/man \ - --infodir=/usr/share/info \ - --datadir=/usr/share \ - --sysconfdir=/etc \ - --localstatedir=/var/lib \ - ${EXTRA_ECONF} \ - ${EECONF_CACHE} \ - "$@" - - if ! ${ECONF_SOURCE}/configure \ - --prefix=/usr \ - --host=${CHOST} \ - --mandir=/usr/share/man \ - --infodir=/usr/share/info \ - --datadir=/usr/share \ - --sysconfdir=/etc \ - --localstatedir=/var/lib \ - ${EXTRA_ECONF} \ - ${EECONF_CACHE} \ - "$@" ; then - - if [ -s config.log ]; then - echo - echo "!!! Please attach the config.log to your bug report:" - echo "!!! ${PWD}/config.log" - fi - die "econf failed" - fi - return $? -} - -strip_duplicate_slashes () -{ - if [ -n "${1}" ]; then - local removed="${1/\/\///}" - [ "${removed}" != "${removed/\/\///}" ] && removed=$(strip_duplicate_slashes "${removed}") - echo ${removed} - fi -} - -einstall() -{ - # CONF_PREFIX is only set if they didn't pass in libdir above - local LOCAL_EXTRA_EINSTALL="${EXTRA_EINSTALL}" - LIBDIR_VAR="LIBDIR_${ABI}" - if [ -n "${ABI}" -a -n "${!LIBDIR_VAR}" ]; then - CONF_LIBDIR="${!LIBDIR_VAR}" - fi - unset LIBDIR_VAR - if [ -n "${CONF_LIBDIR}" ] && [ "${CONF_PREFIX:-unset}" != "unset" ]; then - EI_DESTLIBDIR="${D}/${CONF_PREFIX}/${CONF_LIBDIR}" - EI_DESTLIBDIR="$(strip_duplicate_slashes ${EI_DESTLIBDIR})" - LOCAL_EXTRA_EINSTALL="${LOCAL_EXTRA_EINSTALL} libdir=${EI_DESTLIBDIR}" - unset EI_DESTLIBDIR - fi - - if [ -f ./[mM]akefile -o -f ./GNUmakefile ] ; then - if [ ! -z "${PKGCORE_DEBUG}" ]; then - ${MAKE:-make} -n prefix=${D}/usr \ - datadir=${D}/usr/share \ - infodir=${D}/usr/share/info \ - localstatedir=${D}/var/lib \ - mandir=${D}/usr/share/man \ - sysconfdir=${D}/etc \ - ${LOCAL_EXTRA_EINSTALL} \ - "$@" install - fi - ${MAKE:-make} prefix=${D}/usr \ - datadir=${D}/usr/share \ - infodir=${D}/usr/share/info \ - localstatedir=${D}/var/lib \ - mandir=${D}/usr/share/man \ - sysconfdir=${D}/etc \ - ${LOCAL_EXTRA_EINSTALL} \ - "$@" install || die "einstall failed" - else - die "no Makefile found" - fi -} - -pkg_setup() -{ - return -} - -pkg_nofetch() -{ - [ -z "${SRC_URI}" ] && return - - echo "!!! The following are listed in SRC_URI for ${PN}:" - for MYFILE in `echo ${SRC_URI}`; do - echo "!!! $MYFILE" - done -} - -src_unpack() -{ - if [ "${A}" != "" ]; then - unpack ${A} - fi -} - -src_compile() -{ - if [ -x ./configure ]; then - econf || die "econf failed" - fi - if [ -f Makefile ] || [ -f GNUmakefile ] || [ -f makefile ]; then - emake || die "emake failed" - fi -} - -src_test() -{ - addpredict / - if make check -n &> /dev/null; then - echo ">>> Test phase [check]: ${CATEGORY}/${PF}" - if ! make check; then - hasq test $FEATURES && die "Make check failed. See above for details." - hasq test $FEATURES || eerror "Make check failed. See above for details." - fi - elif make test -n &> /dev/null; then - echo ">>> Test phase [test]: ${CATEGORY}/${PF}" - if ! make test; then - hasq test $FEATURES && die "Make test failed. See above for details." - hasq test $FEATURES || eerror "Make test failed. See above for details." - fi - else - echo ">>> Test phase [none]: ${CATEGORY}/${PF}" - fi - SANDBOX_PREDICT="${SANDBOX_PREDICT%:/}" -} - -src_install() -{ - return -} - -pkg_preinst() -{ - return -} - -pkg_postinst() -{ - return -} - -pkg_prerm() -{ - return -} - -pkg_postrm() -{ - return -} - -into() -{ - if [ $1 == "/" ]; then - export DESTTREE="" - else - export DESTTREE=$1 - if [ ! -d "${D}${DESTTREE}" ]; then - install -d "${D}${DESTTREE}" - fi - fi -} - -insinto() -{ - if [ "$1" == "/" ]; then - export INSDESTTREE="" - else - export INSDESTTREE=$1 - if [ ! -d "${D}${INSDESTTREE}" ]; then - install -d "${D}${INSDESTTREE}" - fi - fi -} - -exeinto() -{ - if [ "$1" == "/" ]; then - export EXEDESTTREE="" - else - export EXEDESTTREE="$1" - if [ ! -d "${D}${EXEDESTTREE}" ]; then - install -d "${D}${EXEDESTTREE}" - fi - fi -} - -docinto() -{ - if [ "$1" == "/" ]; then - export DOCDESTTREE="" - else - export DOCDESTTREE="$1" - if [ ! -d "${D}usr/share/doc/${PF}/${DOCDESTTREE}" ]; then - install -d "${D}usr/share/doc/${PF}/${DOCDESTTREE}" - fi - fi -} - -true diff --git a/pkgcore/bin/ebuild-env/ebuild.sh b/pkgcore/bin/ebuild-env/ebuild.sh deleted file mode 100755 index 029bf3e..0000000 --- a/pkgcore/bin/ebuild-env/ebuild.sh +++ /dev/null @@ -1,771 +0,0 @@ -#!/bin/bash -# ebuild.sh; ebuild phase processing, env handling -# Copyright 2005-2006 Brian Harring <ferringb@gmail.com> -# Copyright 2004-2005 Gentoo Foundation - -# general phase execution path- -# execute_phases is called, which sets EBUILD_PHASE, and then depending on the phase, -# loads or initializes. Env is init'd for non src based stages if the env isn't found- otherwise -# it loads the environ via load_environ call. In cases where env isn't found for phases setup -> merge, -# it bails (theres no way the env should be missing- exemption is setup phase). -# -# for env filtering for restoration and reloading, note the updates to DONT_EXPORT_(VARS|FUNCS). -# those vars are basically used to track what shouldn't be saved/restored. Whitespace seperated, -# those vars can support posix (think egrep) regex. They should hold all vars/funcs that are internal -# ebuild.sh vars. Basically, filter all vars/funcs that are specific to ebuild.sh, not the ebuild. -# -# after loading the env, user defined pre hooks are executed, dyn_${EBUILD_PHASE} is executed, -# and the post hooks are executed. If the env needs to be flushed to disk, MUST_EXPORT_ENV is set to -# "yes", and execute_phases will dump it to ${T}/environment. -# -# few notes on general env stuff- if it's not ebuild specific or a user option, it's typically marked -# readonly. This limits users, but also helps to ensure that reloaded envs from older portages don't -# overwrite an internal ebd.sh function that has since changed. - -ORIG_VARS=`declare | egrep '^[^[:space:]{}()]+=' | cut -s -d '=' -f 1` -ORIG_FUNCS=`declare -F | cut -s -d ' ' -f 3` - -DONT_EXPORT_FUNCS='portageq speak' - -DONT_EXPORT_VARS="ORIG_VARS GROUPS ORIG_FUNCS FUNCNAME DAEMONIZED CCACHE.* DISTCC.* SYNC -\(TMP\|\)DIR FEATURES CONFIG_PROTECT.* P\?WORKDIR RSYNC_.* GENTOO_MIRRORS -\(DIST\|FILES\|RPM\|ECLASS\)DIR HOME MUST_EXPORT_ENV QA_CONTROLLED_EXTERNALLY COLORTERM HOSTNAME -myarg SANDBOX_.* BASH.* EUID PPID SHELLOPTS UID ACCEPT_\(KEYWORDS\|LICENSE\) BUILD\(_PREFIX\|DIR\) T DIRSTACK -DISPLAY \(EBUILD\)\?_PHASE PORTAGE_.* SUDO_.* LD_PRELOAD ret line phases D IMAGE -PORT\(_LOGDIR\|DIR\(_OVERLAY\)\?\) ROOT TERM _ done e PROFILE_.* EBUILD ECLASS LINENO -HILITE TMP HISTCMD OPTIND RANDOM OLDPWD PKGCORE_DOMAIN IFS" - - -if [ -z "$PKGCORE_BIN_PATH" ]; then - echo "PKGCORE_BIN_PATH is unset!" - exit 1 -fi - -# knock the sandbox vars back to the pkgs defaults. -reset_sandbox() { - export SANDBOX_ON="1" - export SANDBOX_PREDICT="${SANDBOX_PREDICT:+${SANDBOX_PREDICT}:}/proc/self/maps:/dev/console:/dev/random:${PORTAGE_TMPDIR}" - export SANDBOX_WRITE="${SANDBOX_WRITE:+${SANDBOX_WRITE}:}/dev/shm:${PORTAGE_TMPDIR}" - export SANDBOX_READ="${SANDBOX_READ:+${SANDBOX_READ}:}/dev/shm:${PORTAGE_TMPDIR}" - local s - for x in CCACHE_DIR DISTCC_DIR D WORKDIR T; do - if [ -n "${!x}" ]; then - addread "${!x}" - addwrite "${!x}" - fi - done -} - -# Prevent aliases from causing portage to act inappropriately. -# Make sure it's before everything so we don't mess aliases that follow. -unalias -a - -# We need this next line for "die" and "assert". It expands -# It _must_ preceed all the calls to die and assert. -shopt -s expand_aliases - -# Unset some variables that break things. -unset GZIP BZIP BZIP2 CDPATH GREP_OPTIONS GREP_COLOR GLOB_IGNORE - -alias die='diefunc "$FUNCNAME" "$LINENO" "$?"' -alias assert='_pipestatus="${PIPESTATUS[*]}"; [[ "${_pipestatus// /}" -eq 0 ]] || diefunc "$FUNCNAME" "$LINENO" "$_pipestatus"' -alias save_IFS='[ "${IFS:-unset}" != "unset" ] && portage_old_IFS="${IFS}"' -alias restore_IFS='if [ "${portage_old_IFS:-unset}" != "unset" ]; then IFS="${portage_old_IFS}"; unset portage_old_IFS; else unset IFS; fi' - -diefunc() { - set +x - # if we were signaled to die... - if [[ -n $EBD_DISABLE_DIEFUNC ]]; then - return - fi - local funcname="$1" lineno="$2" exitcode="$3" - shift 3 - echo >&2 - echo "!!! ERROR: $CATEGORY/$PF failed." >&2 - dump_trace 2 >&2 - echo "!!! ${*:-(no error message)}" >&2 - echo "!!! If you need support, post the topmost build error, NOT this status message." >&2 - if [ "${EBUILD_PHASE/depend}" == "${EBUILD_PHASE}" ]; then - for x in ${EBUILD_DEATH_HOOKS}; do - ${x} ${1} ${2} ${3} "${@}" >&2 1>&2 - done - fi - echo >&2 - exit 1 -} - - -shopt -s extdebug &> /dev/null - -# usage- first arg is the number of funcs on the stack to ignore. -# defaults to 1 (ignoring dump_trace) -dump_trace() { - local funcname="" sourcefile="" lineno="" n e s="yes" - - declare -i strip=1 - - if [[ -n $1 ]]; then - strip=$(( $1 )) - fi - - echo "Call stack:" - for (( n = ${#FUNCNAME[@]} - 1, p = ${#BASH_ARGV[@]} ; n > $strip ; n-- )) ; do - funcname=${FUNCNAME[${n} - 1]} - sourcefile=$(basename ${BASH_SOURCE[${n}]}) - lineno=${BASH_LINENO[${n} - 1]} - # Display function arguments - args= - if [[ -n "${BASH_ARGV[@]}" ]]; then - for (( j = 0 ; j < ${BASH_ARGC[${n} - 1]} ; ++j )); do - newarg=${BASH_ARGV[$(( p - j - 1 ))]} - args="${args:+${args} }'${newarg}'" - done - (( p -= ${BASH_ARGC[${n} - 1]} )) - fi - echo " ${sourcefile}, line ${lineno}: Called ${funcname}${args:+ ${args}}" - done -} - -hasq() { - local x - - local me=$1 - shift - - # All the TTY checks really only help out depend. Which is nice. - # Logging kills all this anyway. Everything becomes a pipe. --NJ - for x in "$@"; do - if [ "${x}" == "${me}" ]; then - return 0 - fi - done - return 1 -} - -hasv() { - if hasq "$@"; then - echo "${1}" - return 0 - fi - return 1 -} - -#if no perms are specified, dirs/files will have decent defaults -#(not secretive, but not stupid) -umask 022 - -# the sandbox is disabled by default except when overridden in the relevant stages -export SANDBOX_ON="0" - -escape_regex() { - local f - while [ -n "$1" ]; do - f="${1//+/\+}" - f="${f//.*/[A-Za-z0-9_-+./]*}" - echo -n "$f" - shift - done -} - -filter_env_func_filter() { - while [ -n "$1" ]; do - echo -n "$(escape_regex "$1")" - [ "$#" != 1 ] && echo -n ',' - shift - done -} - -gen_regex_func_filter() { - local f - if [ "$#" == 1 ]; then - echo -n "$(escape_regex "$1")" - return - fi - echo -n "\($(escape_regex "$1")" - shift - while [ -n "$1" ]; do - echo -n "\|$(escape_regex "$1")" - shift - done - echo -n "\)" -} - -filter_env_var_filter() { - local _internal_var - while [ -n "$1" ]; do - echo -n "$1" - [ "$#" != 1 ] && echo -n ',' - shift - done -} - -gen_regex_var_filter() { - local _internal_var - if [ "$#" == 1 ]; then - echo -n "$1" - return - fi - echo -n "\($1" - shift - while [ -n "$1" ]; do - echo -n "\|$1" - shift - done - echo -n '\)' -} - -# func for beeping and delaying a defined period of time. -sleepbeep() { - if [ ! "$#" -lt 3 ] || [ ! "$#" -gt 0 ]; then - echo "sleepbeep requires one arg- number of beeps" - echo "additionally, can supply a 2nd arg- interval between beeps (defaults to 0.25s" - die "invalid call to sleepbeep" - fi - local count=$(($1)) - local interval="${2:-0.25}" - while [ $count -gt 0 ]; do - echo -en "\a"; - sleep $interval &> /dev/null - count=$(($count - 1)) - done - return 0 -} - -# selectively saves the environ- specifically removes things that have been marked to not be exported. -# dump the environ to stdout. -dump_environ() { - local x y; - - #env dump, if it doesn't match a var pattern, stop processing, else print only if - #it doesn't match one of the filter lists. - # vars, then funcs. - - local opts="" - - [[ $PKGCORE_DEBUG -ge 3 ]] && opts="$opts --debug" - - declare | PYTHONPATH="${PKGCORE_PYTHONPATH}" "${PKGCORE_PYTHON}" \ - "${PKGCORE_BIN_PATH}/filter-env" $opts -f \ - "$(filter_env_func_filter ${DONT_EXPORT_FUNCS} )" -v \ - "$(filter_env_var_filter ${DONT_EXPORT_VARS} f x )" - - if ! hasq "--no-attributes" "$@"; then - echo "# env attributes" - # leave this form so that it's easier to add others in. - for y in export ; do - x=$(${y} | sed -n "/declare \(-[^ ]\+ \)*/!d; s:^declare \(-[^ ]\+ \)*\([A-Za-z0-9_+]\+\)\(=.*$\)\?$:\2:; /^$(gen_regex_var_filter ${DONT_EXPORT_VARS} x y)$/! p;") - [ -n "$x" ] && echo "${y} $(echo $x);" - done - - # if it's just declare -f some_func, filter it, else drop it if it's one of the filtered funcs - declare -F | sed -n "/^declare -[^ ]\( \|[^ ]? $(gen_regex_func_filter ${DONT_EXPORT_FUNCS})$\)\?/d; s/^/ /;s/;*$/;/p;" - - shopt -p - fi -} - -# dump environ to $1, optionally piping it through $2 and redirecting $2's output to $1. -export_environ() { - local temp_umask - if [ "${1:-unset}" == "unset" ]; then - die "export_environ requires at least one arguement" - fi - - #the spaces on both sides are important- otherwise, the later ${DONT_EXPORT_VARS/ temp_umask /} won't match. - #we use spaces on both sides, to ensure we don't remove part of a variable w/ the same name- - # ex: temp_umask_for_some_app == _for_some_app. - #Do it with spaces on both sides. - - DONT_EXPORT_VARS="${DONT_EXPORT_VARS} temp_umask " - temp_umask=`umask` - umask 0002 - - if [ "${2:-unset}" == "unset" ]; then - dump_environ > "$1" - else - dump_environ | $2 > "$1" - fi - chown portage:portage "$1" &>/dev/null - chmod 0664 "$1" &>/dev/null - - DONT_EXPORT_VARS="${DONT_EXPORT_VARS/ temp_umask /}" - - umask $temp_umask -} - -# reload a saved env, applying usual filters to the env prior to eval'ing it. -load_environ() { - local src e ret EXISTING_PATH - # localize these so the reload doesn't have the ability to change them - local DONT_EXPORT_VARS="${DONT_EXPORT_VARS} src e ret" - local DONT_EXPORT_FUNCS="${DONT_EXPORT_FUNCS} load_file declare" - local SANDBOX_STATE=$SANDBOX_ON - local EBUILD_PHASE=$EBUILD_PHASE - local reload_failure=0 - SANDBOX_ON=0 - - SANDBOX_READ="/bin:${SANDBOX_READ}:/dev/urandom:/dev/random:$PKGCORE_BIN_PATH" - SANDBOX_ON=$SANDBOX_STATE - - [ ! -f "$1" ] && die "load_environ called with a nonexist env: $1" - - if [ -z "$1" ]; then - die "load_environ called with no args, need args" - fi - src="$1" - - EXISTING_PATH=$PATH - PKGCORE_ATTRS_EXPORTED= - PKGCORE_ATTRS_READONLY= - PKGCORE_SHOPTS_SET= - PKGCORE_SHOPTS_UNSET= - if [ -f "$src" ]; then - # other managers shove the export/declares inline; we store it in a - # func so that the var attrs can be dropped if needed. - # thus we define these temporarily, to intercept the inlined statements - # and push them into a func. - function declare() { - local r e vars - while [ "${1:0:1}" == "-" ]; do - if [ "${1/r}" != "$1" ]; then - r=1 - fi - if [ "${1/x}" != "$1" ]; then - e=1 - fi - shift - done - if [ -z "$r" ] && [ -z "$e" ]; then - return - fi - while [ -n "$1" ]; do - vars="${vars} ${1/=*}" - shift - done - if [ -n "$r" ]; then - PKGCORE_ATTRS_READONLY="${PKGCORE_ATTRS_READONLY} ${vars}" - fi - if [ -n "$e" ]; then - PKGCORE_ATTRS_EXPORTED="${PKGCORE_ATTRS_EXPORTED} ${vars}" - fi - }; - function export() { - declare -x "$@" - }; - function readonly() { - declare -r "$@" - }; - function shopt() { - if [ "$1" == "-s" ]; then - shift - PKGCORE_SHOPTS_SET="${PKGCORE_SHOPTS_SET} $*" - elif [ "$1" == "-u" ]; then - shift - PKGCORE_SHOPTS_UNSET="${PKGCORE_SHOPTS_UNSET} $*" - else - echo "ignoring unexpected shopt arg in env dump- $*" >&2 - fi - } - local opts="" - [[ $PKGCORE_DEBUG -ge 3 ]] && opts="$opts --debug" - - # run the filtered env. - eval "$(PYTHONPATH=${PKGCORE_PYTHONPATH} \ - "${PKGCORE_PYTHON}" "${PKGCORE_BIN_PATH}/filter-env" $opts \ - -f "$(filter_env_func_filter ${DONT_EXPORT_FUNCS} )" \ - -v "$(filter_env_var_filter ${DONT_EXPORT_VARS} f x EXISTING_PATH)" -i "$src")" - ret=$? - - # if reinstate_loaded_env_attributes exists, run it to add to the vars. - type reinstate_loaded_env_attributes &> /dev/null && \ - reinstate_loaded_env_attributes - unset -f declare readonly export reinstate_loaded_env_attributes shopt - - # do not export/readonly an attr that is filtered- those vars are internal/protected, - # thus their state is guranteed - # additionally, if the var *was* nonexistant, export'ing it serves to create it - - pkgcore_tmp_func() { - while [ -n "$1" ]; do - echo "$1" - shift - done - } - - filter="^$(gen_regex_var_filter $DONT_EXPORT_VARS XARGS)$" - # yes we're intentionally ignoring PKGCORE_ATTRS_READONLY. readonly isn't currently used. - PKGCORE_ATTRS_EXPORTED=$(echo $(pkgcore_tmp_func $PKGCORE_ATTRS_EXPORTED | grep -v "$filter")) - unset pkgcore_tmp_func filter - - # rebuild the func. - local body= - [ -n "$PKGCORE_ATTRS_EXPORTED" ] && body="export $PKGCORE_ATTRS_EXPORTED;" - [ -n "$PKGCORE_SHOPTS_SET" ] && body="${body} shopt -s ${PKGCORE_SHOPTS_SET};" - [ -n "$PKGCORE_SHOPTS_UNSET" ] && body="${body} shopt -u ${PKGCORE_SHOPTS_UNSET};" - unset PKGCORE_ATTRS_READONLY PKGCORE_ATTRS_EXPORTED PKGCORE_SHOPTS_UNSET PKGCORE_SHOPTS_SET - - # and... finally make the func. - eval "reinstate_loaded_env_attributes() { ${body:-:;} };" - else - echo "ebuild=${EBUILD}, phase $EBUILD_PHASE" >&2 - ret=1 - fi - pkgcore_ensure_PATH "$EXISTING_PATH" - return $(( $ret )) -} - -# ensure the passed in PATH has its components in $PATH -pkgcore_ensure_PATH() -{ - local EXISTING_PATH="$1" - local adds - # note this isolates the adds in the same order they appear in - # the passed in path, maintaining that order. - if [ "$EXISTING_PATH" != "$PATH" ]; then - save_IFS - IFS=':' - for x in ${EXISTING_PATH}; do - # keep in mind PATH=":foon" is a valid way to say "cwd" - [ -z "${x}" ] && continue - if ! hasq ${x} ${PATH} && ! hasq ${x} ${adds}; then - adds="${adds:+${adds}:}${x}" - fi - done - restore_IFS - [ -n "$adds" ] && PATH="${PATH}${PATH:+:}${adds}" - export PATH - fi - export PATH -} - -# walk the cascaded profile src'ing it's various bashrcs. -# overriden by daemon normally. -source_profiles() { - local dir - save_IFS - # XXX: Given the following unset, is this set needed? - IFS=$'\n' - for dir in ${PROFILE_PATHS}; do - # Must unset it so that it doesn't mess up assumptions in the RCs. - unset IFS - if [ -f "${dir}/profile.bashrc" ]; then - source "${dir}/profile.bashrc" - fi - done - restore_IFS - if [ -f "$PORTAGE_BASHRC" ]; then - source "$PORTAGE_BASHRC" - fi -} - -# do all profile, bashrc's, and ebuild sourcing. Should only be called in setup phase, unless the -# env is *completely* missing, as it is occasionally for ebuilds during prerm/postrm. -init_environ() { - OCC="$CC" - OCXX="$CXX" - local EXISTING_PATH="$PATH" - - if [ "${EBUILD_PHASE}" == "setup" ]; then - #we specifically save the env so it's not stomped on by sourcing. - #bug 51552 - dump_environ --no-attributes > "${T}/.temp_env" - - if [ "$USERLAND" == "GNU" ]; then - local PORTAGE_SHIFTED_PATH="$PATH" - source /etc/profile.env &>/dev/null - fi - - #restore the saved env vars. - if ! load_environ "${T}/.temp_env"; then - #this shouldn't happen. - die "failed to load ${T}/.tmp_env- fs is readonly?" - fi - - rm "${T}/.temp_env" - source_profiles - fi - - if [ "${EBUILD_PHASE}" != "depend" ]; then - [ ! -z "$OCC" ] && export CC="$OCC" - [ ! -z "$OCXX" ] && export CXX="$OCXX" - - fi - - # if daemonized, it's already loaded these funcs. - if [ "$DAEMONIZED" != "yes" ]; then - source "${PKGCORE_BIN_PATH}/ebuild-functions.sh" || die "failed sourcing ebuild-functions.sh" - fi - SANDBOX_ON="1" - export S=${WORKDIR}/${P} - - # Expand KEYWORDS - # We need to turn off pathname expansion for -* in KEYWORDS and - # we need to escape ~ to avoid tilde expansion (damn bash) :) - set -f - KEYWORDS="$(echo ${KEYWORDS//~/\\~})" - set +f - - unset IUSE DEPEND RDEPEND CDEPEND PDEPEND - unset E_IUSE E_DEPEND E_RDEPEND E_CDEPEND E_PDEPEND - - if [ ! -f "${EBUILD}" ]; then - echo "bailing, ebuild not found at '$EBUILD'" - die "EBUILD=${EBUILD}; problem is, it doesn't exist. bye." >&2 - fi - - # XXX: temp hack to make misc broken eclasses behave, java-utils-2 for example - # XXX: as soon as these eclasses behave, remove this. - export DESTTREE=/usr - - source "${EBUILD}" - if [ "${EBUILD_PHASE}" != "depend" ]; then - RESTRICT="${FINALIZED_RESTRICT}" - unset FINALIZED_RESTRICT - fi - - [ -z "${ERRORMSG}" ] || die "${ERRORMSG}" - - hasq nostrip ${RESTRICT} && export DEBUGBUILD=1 - - #a reasonable default for $S - if [ "$S" = "" ]; then - export S=${WORKDIR}/${P} - fi - - #some users have $TMP/$TMPDIR to a custom dir in their home ... - #this will cause sandbox errors with some ./configure - #scripts, so set it to $T. - export TMP="${T}" - export TMPDIR="${T}" - - # Note: this next line is not the same as export RDEPEND=${RDEPEND:-${DEPEND}} - # That will test for unset *or* NULL (""). We want just to set for unset... - - if [ "${RDEPEND-unset}" == "unset" ]; then - export RDEPEND="${DEPEND}" - fi - - #add in dependency info from eclasses - IUSE="$IUSE $E_IUSE" - DEPEND="${DEPEND} ${E_DEPEND}" - RDEPEND="$RDEPEND $E_RDEPEND" - CDEPEND="$CDEPEND $E_CDEPEND" - PDEPEND="$PDEPEND $E_PDEPEND" - - unset E_IUSE E_DEPEND E_RDEPEND E_CDEPEND E_PDEPEND - pkgcore_ensure_PATH "$EXISTING_PATH" -} - -# short version. think these should be sourced via at the daemons choice, rather then defacto. -source "${PKGCORE_BIN_PATH}/ebuild-default-functions.sh" || die "failed sourcing ebuild-default-functions.sh" -source "${PKGCORE_BIN_PATH}/isolated-functions.sh" || die "failed sourcing stripped down functions.sh" - -# general func to call for phase execution. this handles necessary env loading/dumping, and executing pre/post/dyn -# calls. -execute_phases() { - local ret - for myarg in $*; do - EBUILD_PHASE="$myarg" - MUST_EXPORT_ENV="no" - case $EBUILD_PHASE in - nofetch) - init_environ - pkg_nofetch - ;; - prerm|postrm|preinst|postinst|config) - export SANDBOX_ON="0" - - if ! load_environ "${T}/environment"; then - #hokay. this sucks. - ewarn - ewarn "failed to load env" - ewarn "this installed pkg may not behave correctly" - ewarn - sleepbeep 10 - fi - - [[ $PKGCORE_DEBUG -ge 3 ]] && set -x - if type reinstate_loaded_env_attributes &> /dev/null; then - reinstate_loaded_env_attributes - unset -f reinstate_loaded_env_attributes - fi - [[ -n $PKGCORE_DEBUG ]] && set -x - type -p pre_pkg_${EBUILD_PHASE} &> /dev/null && pre_pkg_${EBUILD_PHASE} - if type -p dyn_${EBUILD_PHASE}; then - dyn_${EBUILD_PHASE} - else - pkg_${EBUILD_PHASE} - fi - ret=0 - - type -p post_pkg_${EBUILD_PHASE} &> /dev/null && post_pkg_${EBUILD_PHASE} - [[ $PKGCORE_DEBUG -lt 2 ]] && set +x - ;; - unpack|compile|test|install) - if [ "${SANDBOX_DISABLED="0"}" == "0" ]; then - export SANDBOX_ON="1" - else - export SANDBOX_ON="0" - fi - - [[ $PKGCORE_DEBUG -ge 3 ]] && set -x - if ! load_environ ${T}/environment; then - ewarn - ewarn "failed to load env. This is bad, bailing." - die "unable to load saved env for phase $EBUILD_PHASE, unwilling to continue" - fi - if type reinstate_loaded_env_attributes &> /dev/null; then - reinstate_loaded_env_attributes - unset -f reinstate_loaded_env_attributes - fi - [[ -n $PKGCORE_DEBUG ]] && set -x - type -p pre_src_${EBUILD_PHASE} &> /dev/null && pre_src_${EBUILD_PHASE} - dyn_${EBUILD_PHASE} - ret=0 - type -p post_src_${EBUILD_PHASE} &> /dev/null && post_src_${EBUILD_PHASE} - [[ $PKGCORE_DEBUG -lt 2 ]] && set +x - export SANDBOX_ON="0" - ;; - setup|setup-binpkg) - #pkg_setup needs to be out of the sandbox for tmp file creation; - #for example, awking and piping a file in /tmp requires a temp file to be created - #in /etc. If pkg_setup is in the sandbox, both our lilo and apache ebuilds break. - - export SANDBOX_ON="0" - - # binpkgs don't need to reinitialize the env. - if [ "$myarg" == "setup" ]; then - [ ! -z "${DISTCC_LOG}" ] && addwrite "$(dirname ${DISTCC_LOG})" - - local x - # if they aren't set, then holy hell ensues. deal. - - [ -z "${CCACHE_SIZE}" ] && export CCACHE_SIZE="500M" - ccache -M ${CCACHE_SIZE} &> /dev/null - [[ $PKGCORE_DEBUG == 2 ]] && set -x - init_environ - MUST_EXPORT_ENV="yes" - elif ! load_environ ${T}/environment; then - die "failed loading saved env; at ${T}/environment" - fi - - [[ -n $PKGCORE_DEBUG ]] && set -x - type -p pre_pkg_setup &> /dev/null && \ - pre_pkg_setup - dyn_setup - ret=0; - type -p post_pkg_setup &> /dev/null && \ - post_pkg_setup - [[ $PKGCORE_DEBUG -lt 2 ]] && set +x - - ;; - depend) - SANDBOX_ON="1" - MUST_EXPORT_ENV="no" - - if [ -z "$QA_CONTROLLED_EXTERNALLY" ]; then - enable_qa_interceptors - fi - - init_environ - - if [ -z "$QA_CONTROLLED_EXTERNALLY" ]; then - disable_qa_interceptors - fi - - speak "$(pkgcore_dump_metadata_keys)" - ;; - *) - export SANDBOX_ON="1" - echo "Please specify a valid command: $EBUILD_PHASE isn't valid." - echo - dyn_help - exit 1 - ;; - esac - - if [ "${MUST_EXPORT_ENV}" == "yes" ]; then - export_environ "${T}/environment" - MUST_EXPORT_ENV="no" - fi - [[ $PKGCORE_DEBUG -lt 4 ]] && set +x - done - return ${ret:-0} -} - -pkgcore_dump_metadata_keys() { - set -f - [ "${DEPEND:-unset}" != "unset" ] && echo "key DEPEND=$(echo $DEPEND)" - [ "${RDEPEND:-unset}" != "unset" ] && echo "key RDEPEND=$(echo $RDEPEND)" - [ "$SLOT:-unset}" != "unset" ] && echo "key SLOT=$(echo $SLOT)" - [ "$SRC_URI:-unset}" != "unset" ] && echo "key SRC_URI=$(echo $SRC_URI)" - [ "$RESTRICT:-unset}" != "unset" ] && echo "key RESTRICT=$(echo $RESTRICT)" - [ "$HOMEPAGE:-unset}" != "unset" ] && echo "key HOMEPAGE=$(echo $HOMEPAGE)" - [ "$LICENSE:-unset}" != "unset" ] && echo "key LICENSE=$(echo $LICENSE)" - [ "$DESCRIPTION:-unset}" != "unset" ] && echo "key DESCRIPTION=$(echo $DESCRIPTION)" - [ "$KEYWORDS:-unset}" != "unset" ] && echo "key KEYWORDS=$(echo $KEYWORDS)" - [ "$INHERITED:-unset}" != "unset" ] && echo "key INHERITED=$(echo $INHERITED)" - [ "$IUSE:-unset}" != "unset" ] && echo "key IUSE=$(echo $IUSE)" - [ "$CDEPEND:-unset}" != "unset" ] && echo "key CDEPEND=$(echo $CDEPEND)" - [ "$PDEPEND:-unset}" != "unset" ] && echo "key PDEPEND=$(echo $PDEPEND)" - [ "$PROVIDE:-unset}" != "unset" ] && echo "key PROVIDE=$(echo $PROVIDE)" - [ "$EAPI:-unset}" != "unset" ] && echo "key EAPI=$(echo $EAPI)" - set +f -} - -#echo, everything has been sourced. now level the read-only's. -if [ "$*" != "daemonize" ]; then - for x in ${DONT_EXPORT_FUNCS}; do - declare -fr "$x" - done - unset x -fi - -f="$(declare | { - read l; - while [ "${l% \(\)}" == "$l" ]; do - echo "${l/=*}"; - read l; - done; - unset l - })" - -#update the don't export filters. -if [ -z "${ORIG_VARS}" ]; then - DONT_EXPORT_VARS="${DONT_EXPORT_VARS} ${f}" -else - DONT_EXPORT_VARS="${DONT_EXPORT_VARS} $(echo "${f}" | grep -v "^$(gen_regex_var_filter ${ORIG_VARS})\$")" -fi -unset f - -[ -z "${ORIG_FUNCS}" ] && DONT_EXPORT_FUNCS="${DONT_EXPORT_FUNCS} $(declare -F | cut -s -d ' ' -f 3)" -set +f - -export XARGS -if [ "$(id -nu)" == "portage" ] ; then - export USER=portage -fi -set +H -h -# if we're being src'd for our functions, do nothing. if called directly, define a few necessary funcs. -if [ "$*" != "daemonize" ]; then - - if [ "${*/depend}" != "$*" ]; then - speak() { - echo "$*" >&4 - } - declare -rf speak - fi - if [ -z "${NOCOLOR}" ]; then - set_colors - else - unset_colors - fi - unset x - execute_phases $* - exit 0 -else - DAEMONIZED="yes" - export DAEMONIZED - readonly DAEMONIZED -fi -: diff --git a/pkgcore/bin/ebuild-env/filter-env b/pkgcore/bin/ebuild-env/filter-env deleted file mode 100755 index 455e337..0000000 --- a/pkgcore/bin/ebuild-env/filter-env +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env python - -"""Commandline wrapper.""" - -from pkgcore.scripts import filter_env -from pkgcore.util import commandline - -if __name__ == '__main__': - commandline.main({None: (filter_env.OptionParser, filter_env.main)}) diff --git a/pkgcore/bin/ebuild-env/isolated-functions.sh b/pkgcore/bin/ebuild-env/isolated-functions.sh deleted file mode 100644 index ee5d690..0000000 --- a/pkgcore/bin/ebuild-env/isolated-functions.sh +++ /dev/null @@ -1,187 +0,0 @@ -# Copyright 1999-2006 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 -# $Header$ - -# Internal logging function, don't use this in ebuilds -elog_base() { - local messagetype - [ -z "${1}" -o -z "${T}" -o ! -d "${T}/logging" ] && return 1 - case "${1}" in - INFO|WARN|ERROR|LOG) - messagetype="${1}" - shift - ;; - *) - echo -e " ${PKGCORE_RC_BAD}*${PKGCORE_RC_NORMAL} Invalid use of internal function elog_base(), next message will not be logged" - return 1 - ;; - esac - echo "$*" >> ${T}/logging/${EBUILD_PHASE}.${messagetype} - return 0 -} - -elog() { - elog_base LOG "$*" - echo -e " ${PKGCORE_RC_GOOD}*${PKGCORE_RC_NORMAL} $*" - return 0 -} - -esyslog() { - local pri= - local tag= - - if [ -x /usr/bin/logger ] - then - pri="$1" - tag="$2" - - shift 2 - [ -z "$*" ] && return 0 - - /usr/bin/logger -p "${pri}" -t "${tag}" -- "$*" - fi - - return 0 -} - -einfo() { - einfon "$*\n" - PKGCORE_RC_LAST_CMD="einfo" - return 0 -} - -einfon() { - elog_base INFO "$*" - echo -ne " ${PKGCORE_RC_GOOD}*${PKGCORE_RC_NORMAL} $*" - PKGCORE_RC_LAST_CMD="einfon" - return 0 -} - -ewarn() { - elog_base WARN "$*" - echo -e " ${PKGCORE_RC_WARN}*${PKGCORE_RC_NORMAL} $*" - PKGCORE_RC_LAST_CMD="ewarn" - return 0 -} - -eerror() { - elog_base ERROR "$*" - echo -e " ${PKGCORE_RC_BAD}*${PKGCORE_RC_NORMAL} $*" - PKGCORE_RC_LAST_CMD="eerror" - return 0 -} - -ebegin() { - local msg="$* ..." - einfon "${msg}" - echo - PKGCORE_RC_LAST_CMD="ebegin" - return 0 -} - -_eend() { - local retval=${1:-0} efunc=${2:-eerror} msg - shift 2 - - if [[ ${retval} == "0" ]] ; then - msg="${PKGCORE_RC_BRACKET}[ ${PKGCORE_RC_GOOD}ok${PKGCORE_RC_BRACKET} ]${PKGCORE_RC_NORMAL}" - else - if [[ -n $* ]] ; then - ${efunc} "$*" - fi - msg="${PKGCORE_RC_BRACKET}[ ${PKGCORE_RC_BAD}!!${PKGCORE_RC_BRACKET} ]${PKGCORE_RC_NORMAL}" - fi - - echo -e "${PKGCORE_RC_ENDCOL} ${msg}" - - return ${retval} -} - -eend() { - local retval=${1:-0} - shift - - _eend ${retval} eerror "$*" - - return ${retval} -} - -KV_major() { - [[ -z $1 ]] && return 1 - - local KV=$@ - echo "${KV%%.*}" -} - -KV_minor() { - [[ -z $1 ]] && return 1 - - local KV=$@ - KV=${KV#*.} - echo "${KV%%.*}" -} - -KV_micro() { - [[ -z $1 ]] && return 1 - - local KV=$@ - KV=${KV#*.*.} - echo "${KV%%[^[:digit:]]*}" -} - -KV_to_int() { - [[ -z $1 ]] && return 1 - - local KV_MAJOR=$(KV_major "$1") - local KV_MINOR=$(KV_minor "$1") - local KV_MICRO=$(KV_micro "$1") - local KV_int=$(( KV_MAJOR * 65536 + KV_MINOR * 256 + KV_MICRO )) - - # We make version 2.2.0 the minimum version we will handle as - # a sanity check ... if its less, we fail ... - if [[ ${KV_int} -ge 131584 ]] ; then - echo "${KV_int}" - return 0 - fi - - return 1 -} - -get_KV() { - echo $(KV_to_int "$(uname -r)") -} - -unset_colors() { - PKGCORE_RC_COLS="25 80" - PKGCORE_RC_ENDCOL= - PKGCORE_RC_GOOD= - PKGCORE_RC_WARN= - PKGCORE_RC_BAD= - PKGCORE_RC_NORMAL= - PKGCORE_RC_HILITE= - PKGCORE_RC_BRACKET= -} - -set_colors() { - # try setting the column width to bash's internal COLUMNS variable, - # then try to get it via stty. no go? hardcode it to 80. - PKGCORE_RC_COLS=${COLUMNS:-0} - (( PKGCORE_RC_COLS == 0 )) && PKGCORE_RC_COLS=$(set -- `stty size 2>/dev/null` ; echo $2) - (( PKGCORE_RC_COLS > 0 )) || (( PKGCORE_RC_COLS = 80 )) - PKGCORE_RC_COLS=$((${PKGCORE_RC_COLS} - 8)) # width of [ ok ] == 7 - - PKGCORE_RC_ENDCOL=$'\e[A\e['${PKGCORE_RC_COLS}'C' - # Now, ${PKGCORE_RC_ENDCOL} will move us to the end of the - # column; irregardless of character width - - PKGCORE_RC_GOOD=$'\e[32;01m' - PKGCORE_RC_WARN=$'\e[33;01m' - PKGCORE_RC_BAD=$'\e[31;01m' - PKGCORE_RC_HILITE=$'\e[36;01m' - PKGCORE_RC_BRACKET=$'\e[34;01m' - PKGCORE_RC_NORMAL=$'\e[0m' -} - -unset_colors -DONT_EXPORT_VARS="${DONT_EXPORT_VARS} PKGCORE_RC_.*" -true diff --git a/pkgcore/bin/ebuild-env/portageq_emulation b/pkgcore/bin/ebuild-env/portageq_emulation deleted file mode 100755 index 8d644bc..0000000 --- a/pkgcore/bin/ebuild-env/portageq_emulation +++ /dev/null @@ -1,178 +0,0 @@ -#!/usr/bin/python -O -# Copyright 2006 Brian Harring <ferringb@gmail.com> - -# disable sandbox for any pyc regens -import os -env = os.environ["SANDBOX_ON"] = "0" - -from snakeoil.demandload import demandload -demandload(globals(), - "snakeoil.iterables:caching_iter", - "pkgcore.config:load_config", - "pkgcore.ebuild.atom:atom", - "pkgcore.util.packages:get_raw_pkg", - "sys", - "os", -) - -def str_pkg(pkg): - pkg = get_raw_pkg(pkg) - # special casing; old style virtuals come through as the original pkg. - if pkg.package_is_real: - return pkg.cpvstr - if hasattr(pkg, "actual_pkg"): - return pkg.actual_pkg.cpvstr - # icky, but works. - return str(pkg.rdepends).lstrip("=") - -def expose_to_commandline(count, **kwds): - def internal_f(f): - f.args = count - f.swallow_root = kwds.pop("swallow_root", False) - f.command_handler = True - return f - return internal_f - -def set_arg_count(count): - def internal_f(f): - f.args = count - return f - return internal_f - -default_get = lambda d,k: d.settings.get(k, "") -distdir_get = lambda d,k: d.settings["fetcher"].distdir -envvar_getter = {"DISTDIR":distdir_get} - -@expose_to_commandline(-1) -def envvar(domain, *keys): - """ - return configuration defined variables - """ - return ["".join("%s\n" % envvar_getter.get(x, default_get)(domain, x) - for x in keys), 0] - -def make_atom(a): - a = atom(a) - # force expansion. - a.restrictions - return a - -@expose_to_commandline(1, swallow_root=True) -def has_version(domain, arg): - """ - @param domain: L{pkgcore.config.domain.domain} instance - @param atom_str: L{pkgcore.ebuild.atom.atom} instance - """ - arg = make_atom(arg) - if caching_iter(domain.all_vdbs.itermatch(arg)): - return ['', 0] - return ['', 1] - -@expose_to_commandline(-1, swallow_root=True) -def mass_best_version(domain, *args): - """ - multiple best_version calls - """ - return ["".join("%s:%s\n" % (x, best_version(domain, x)[0].rstrip()) - for x in args), 0] - -@expose_to_commandline(1, swallow_root=True) -def best_version(domain, arg): - """ - @param domain: L{pkgcore.config.domain.domain} instance - @param atom_str: L{pkgcore.ebuild.atom.atom} instance - """ - # temp hack, configured pkgs yield "configured(blah) pkg" - arg = make_atom(arg) - try: - p = max(domain.all_vdbs.itermatch(arg)) - except ValueError: - # empty sequence. - return ['', 1] - return [str_pkg(get_raw_pkg(p)) + "\n", 0] - - -@expose_to_commandline(1, swallow_root=True) -def match(domain, arg): - """ - @param domain: L{pkgcore.config.domain.domain} instance - @param atom_str: L{pkgcore.ebuild.atom.atom} instance - """ - arg = make_atom(arg) - # temp hack, configured pkgs yield "configured(blah) pkg" - l = sorted(get_raw_pkg(x) for x in domain.all_repos.itermatch(arg)) - if not l: - return ['', 1] - return ["".join(str_pkg(x) +"\n" for x in l), 0] - - -def usage(): - print "\nusage: command domain atom" - print "domain is the string name of the domain to query from; if exempted, will use the default domain" - print "\n=available commands=\n" - for k, v in globals().iteritems(): - if not getattr(v, "command_handler", False): - continue - print k - print "\n".join(" "+x for x in [s.strip() for s in v.__doc__.split("\n")] if x) - print - -def main(): - a = sys.argv[1:] - if "--usage" in a or "--help" in a: - usage() - sys.exit(0) - if not a: - usage() - sys.exit(1) - - if "--domain" in a: - i = a.index("--domain") - domain = a[i+1] - del a[i] - del a[i] - else: - domain = None - try: - command = globals()[a[0]] - if not getattr(command, "command_handler", False): - raise KeyError - except KeyError: - print "%s isn't a valid command" % a[0] - usage() - sys.exit(2) - - if command.swallow_root: - try: - a.pop(0) - except IndexError: - print "arg count is wrong" - usage() - sys.exit(2) - - bad = False - if command.args == -1: - bad = not a - else: - bad = len(a) - 1 != command.args - if bad: - print "arg count is wrong" - usage() - sys.exit(2) - - if domain is None: - domain = load_config().get_default("domain") - else: - domain = load_config().domain.get(domain) - - if domain is None: - print "no default domain in your configuration, or what was specified manually wasn't found." - print "known domains- %r" % list(load_config().domain.iterkeys()) - sys.exit(2) - - s, ret = command(domain, *a[1:]) - sys.stdout.write(s) - sys.exit(ret) - -if __name__ == "__main__": - main() diff --git a/pkgcore/bin/ebuild-helpers/dobin b/pkgcore/bin/ebuild-helpers/dobin deleted file mode 100755 index a3269ed..0000000 --- a/pkgcore/bin/ebuild-helpers/dobin +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash -# Copyright 1999-2006 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 -# $Id: /var/cvsroot/gentoo-src/portage/bin/dobin,v 1.13 2004/10/04 13:56:50 vapier Exp $ - -if [[ $# -lt 1 ]] ; then - echo "$0: at least one argument needed" 1>&2 - exit 1 -fi - -if [[ ! -d ${D}${DESTTREE}/bin ]] ; then - install -d "${D}${DESTTREE}/bin" || exit 2 -fi - -ret=0 - -for x in "$@" ; do - if [[ -e ${x} ]] ; then - install -m0755 -o ${PORTAGE_INST_UID:-0} -g ${PORTAGE_INST_GID:-0} "${x}" "${D}${DESTTREE}/bin" - else - echo "!!! ${0##*/}: ${x} does not exist" 1>&2 - false - fi - ((ret+=$?)) -done - -exit ${ret} diff --git a/pkgcore/bin/ebuild-helpers/doconfd b/pkgcore/bin/ebuild-helpers/doconfd deleted file mode 100755 index e9ea1fd..0000000 --- a/pkgcore/bin/ebuild-helpers/doconfd +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -# Copyright 1999-2006 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 -# $Id: /var/cvsroot/gentoo-src/portage/bin/doconfd,v 1.2.2.1 2005/01/13 04:51:56 vapier Exp $ - -if [[ $# -lt 1 ]] ; then - echo "$0: at least one argument needed" 1>&2 - exit 1 -fi - -exec \ -env \ -INSDESTTREE="/etc/conf.d/" \ -doins "$@" diff --git a/pkgcore/bin/ebuild-helpers/dodir b/pkgcore/bin/ebuild-helpers/dodir deleted file mode 100755 index bc4f7f5..0000000 --- a/pkgcore/bin/ebuild-helpers/dodir +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -# Copyright 1999-2006 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 -# $Id: /var/cvsroot/gentoo-src/portage/bin/dodir,v 1.5 2004/10/04 13:56:50 vapier Exp $ - -slash=/ -exec install -d ${DIROPTIONS} "${@/#${slash}/${D}${slash}}" diff --git a/pkgcore/bin/ebuild-helpers/dodoc b/pkgcore/bin/ebuild-helpers/dodoc deleted file mode 100755 index 60b6a27..0000000 --- a/pkgcore/bin/ebuild-helpers/dodoc +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -# Copyright 1999-2006 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -if [ $# -lt 1 ] ; then - echo "$0: at least one argument needed" 1>&2 - exit 1 -fi - -dir="${D}usr/share/doc/${PF}/${DOCDESTTREE}" -if [ ! -d "${dir}" ] ; then - install -d "${dir}" -fi - -ret=0 -for x in "$@" ; do - if [ -s "${x}" ] ; then - install -m0644 "${x}" "${dir}" - gzip -f -9 "${dir}/${x##*/}" - elif [ ! -e "${x}" ] ; then - echo "dodoc: ${x} does not exist" 1>&2 - ((++ret)) - fi -done - -exit ${ret} diff --git a/pkgcore/bin/ebuild-helpers/doenvd b/pkgcore/bin/ebuild-helpers/doenvd deleted file mode 100755 index 5232ed9..0000000 --- a/pkgcore/bin/ebuild-helpers/doenvd +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -# Copyright 1999-2006 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 -# $Id: /var/cvsroot/gentoo-src/portage/bin/doenvd,v 1.2.2.1 2005/01/13 04:51:56 vapier Exp $ - -if [[ $# -lt 1 ]] ; then - echo "$0: at least one argument needed" 1>&2 - exit 1 -fi - -exec \ -env \ -INSDESTTREE="/etc/env.d/" \ -doins "$@" diff --git a/pkgcore/bin/ebuild-helpers/doexe b/pkgcore/bin/ebuild-helpers/doexe deleted file mode 100755 index 3badead..0000000 --- a/pkgcore/bin/ebuild-helpers/doexe +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash -# Copyright 1999-2006 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 -# $Id: /var/cvsroot/gentoo-src/portage/bin/doexe,v 1.10.2.1 2004/12/06 03:01:43 carpaski Exp $ - - -if [ -z "${PKGCORE_BIN_PATH}" ]; then - echo "PKGCORE_BIN_PATH is unset!" - exit -1 -fi -source "${PKGCORE_BIN_PATH}/isolated-functions.sh" - -if [[ $# -lt 1 ]] ; then - echo "$0: at least one argument needed" 1>&2 - exit 1 -fi - -if [[ ! -d ${D}${EXEDESTTREE} ]] ; then - install -d "${D}${EXEDESTTREE}" -fi - -for x in "$@" ; do - if [ -L "${x}" ] ; then - cp "${x}" "${T}" - mysrc="${T}"/$(/usr/bin/basename "${x}") - elif [ -d "${x}" ] ; then - vecho "doexe: warning, skipping directory ${x}" - continue - else - mysrc="${x}" - fi - install ${EXEOPTIONS} "${mysrc}" "${D}${EXEDESTTREE}" -done diff --git a/pkgcore/bin/ebuild-helpers/dohard b/pkgcore/bin/ebuild-helpers/dohard deleted file mode 100755 index 2270487..0000000 --- a/pkgcore/bin/ebuild-helpers/dohard +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -# Copyright 1999-2007 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -if [[ $# -ne 2 ]] ; then - echo "$0: two arguments needed" 1>&2 - exit 1 -fi - -destdir=${2%/*} -[[ ! -d ${D}${destdir} ]] && dodir "${destdir}" - -exec ln -f "${D}$1" "${D}$2" diff --git a/pkgcore/bin/ebuild-helpers/dohtml b/pkgcore/bin/ebuild-helpers/dohtml deleted file mode 100755 index e5614ab..0000000 --- a/pkgcore/bin/ebuild-helpers/dohtml +++ /dev/null @@ -1,172 +0,0 @@ -#!/usr/bin/python -# Copyright 1999-2006 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 -# $Id: /var/cvsroot/gentoo-src/portage/bin/dohtml,v 1.14.2.1 2004/10/27 14:39:29 jstubbs Exp $ - -# -# Typical usage: -# dohtml -r docs/* -# - put all files and directories in docs into /usr/share/doc/${PF}/html -# dohtml foo.html -# - put foo.html into /usr/share/doc/${PF}/html -# -# -# Detailed usage: -# dohtml <list-of-files> -# - will install the files in the list of files (space-separated list) into -# /usr/share/doc/${PF}/html, provided the file ends in .html, .png, .jpg -# or .css -# dohtml -r <list-of-files-and-directories> -# - will do as 'dohtml', but recurse into all directories, as long as the -# directory name is not CVS -# dohtml -A jpe,java [-r] <list-of-files[-and-directories]> -# - will do as 'dohtml' but add .jpe,.java (default filter list is -# added to your list) -# dohtml -a png,gif,html,htm [-r] <list-of-files[-and-directories]> -# - will do as 'dohtml' but filter on .png,.gif,.html,.htm (default filter -# list is ignored) -# dohtml -x CVS,SCCS,RCS -r <list-of-files-and-directories> -# - will do as 'dohtml -r', but ignore directories named CVS, SCCS, RCS -# - -import os -import string -import sys -import types - -def dodir(path): - os.system("install -d '%s'" % path) - -def dofile(src,dst): - - os.system("install -m0644 '%s' '%s'" % (src, dst)) - -def install(basename, dirname, options, prefix=""): - - fullpath = basename - if prefix: fullpath = prefix + "/" + fullpath - if dirname: fullpath = dirname + "/" + fullpath - - if options.DOCDESTTREE: - destdir = options.D + "usr/share/doc/" + options.PF + "/" + options.DOCDESTTREE + "/" + options.doc_prefix + "/" + prefix - else: - destdir = options.D + "usr/share/doc/" + options.PF + "/html/" + options.doc_prefix + "/" + prefix - - if os.path.isfile(fullpath): - ext = os.path.splitext(basename)[1] - if (len(ext) and ext[1:] in options.allowed_exts) or basename in options.allowed_files: - dodir(destdir) - dofile(fullpath, destdir + "/" + basename) - elif options.recurse and os.path.isdir(fullpath) and \ - basename not in options.disallowed_dirs: - for i in os.listdir(fullpath): - pfx = basename - if prefix: pfx = prefix + "/" + pfx - install(i, dirname, options, pfx) - else: - return False - return True - - -class OptionsClass: - def __init__(self): - self.PF = "" - self.D = "" - self.DOCDESTTREE = "" - - if os.environ.has_key("PF"): - self.PF = os.environ["PF"] - if os.environ.has_key("D"): - self.D = os.environ["D"] - if os.environ.has_key("DOCDESTTREE"): - self.DOCDESTTREE = os.environ["DOCDESTTREE"] - - self.allowed_exts = [ 'png', 'gif', 'html', 'htm', 'jpg', 'css', 'js' ] - self.allowed_files = [] - self.disallowed_dirs = [ 'CVS' ] - self.recurse = False - self.verbose = False - self.doc_prefix = "" - -def print_help(): - opts = OptionsClass() - - print "dohtml [-a .foo,.bar] [-A .foo,.bar] [-f foo,bar] [-x foo,bar]" - print " [-r] [-V] <file> [file ...]" - print - print " -a Set the list of allowed to those that are specified." - print " Default:", string.join(opts.allowed_exts, ",") - print " -A Extend the list of allowed file types." - print " -f Set list of allowed extensionless file names." - print " -x Set directories to be excluded from recursion." - print " Default:", string.join(opts.disallowed_dirs, ",") - print " -r Install files and directories recursively." - print " -V Be verbose." - print - -def parse_args(): - options = OptionsClass() - args = [] - - x = 1 - while x < len(sys.argv): - arg = sys.argv[x] - if arg in ["-h","-r","-V"]: - if arg == "-h": - print_help() - sys.exit(0) - elif arg == "-r": - options.recurse = True - elif arg == "-V": - options.verbose = True - elif sys.argv[x] in ["-A","-a","-f","-x","-p"]: - x += 1 - if x == len(sys.argv): - print_help() - sys.exit(0) - elif arg == "-p": - options.doc_prefix = sys.argv[x] - else: - values = string.split(sys.argv[x], ",") - if arg == "-A": - options.allowed_exts.extend(values) - elif arg == "-a": - options.allowed_exts = values - elif arg == "-f": - options.allowed_files = values - elif arg == "-x": - options.disallowed_dirs = values - else: - args.append(sys.argv[x]) - x += 1 - - return (options, args) - -def main(): - - (options, args) = parse_args() - - if type(options.allowed_exts) == types.StringType: - options.allowed_exts = options.allowed_exts.split(",") - - if options.verbose: - print "Allowed extensions:", options.allowed_exts - print "Document prefix : '" + options.doc_prefix + "'" - print "Allowed files :", options.allowed_files - - success = True - - for x in args: - basename = os.path.basename(x) - dirname = os.path.dirname(x) - success = success and install(basename, dirname, options) - - if success: - retcode = 0 - else: - retcode = 1 - - sys.exit(retcode) - -if __name__ == "__main__": - main() diff --git a/pkgcore/bin/ebuild-helpers/doinfo b/pkgcore/bin/ebuild-helpers/doinfo deleted file mode 100755 index 67b163b..0000000 --- a/pkgcore/bin/ebuild-helpers/doinfo +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash -# Copyright 1999-2006 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 -# $Id: /var/cvsroot/gentoo-src/portage/bin/doinfo,v 1.7 2004/10/04 13:56:50 vapier Exp $ - -if [ ${#} -lt 1 ] ; then - echo "doinfo: at least one argument needed" - exit 1 -fi -if [ ! -d "${D}usr/share/info" ] ; then - install -d "${D}usr/share/info" -fi - -for x in "$@" ; do - if [ -e "${x}" ] ; then - install -m0644 "${x}" "${D}usr/share/info" - gzip -f -9 "${D}usr/share/info/${x##*/}" - else - echo "doinfo: ${x} does not exist" - fi -done diff --git a/pkgcore/bin/ebuild-helpers/doinitd b/pkgcore/bin/ebuild-helpers/doinitd deleted file mode 100755 index 8aae1b3..0000000 --- a/pkgcore/bin/ebuild-helpers/doinitd +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -# Copyright 1999-2006 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 -# $Id: /var/cvsroot/gentoo-src/portage/bin/doinitd,v 1.2.2.1 2005/01/13 04:51:56 vapier Exp $ - -if [[ $# -lt 1 ]] ; then - echo "$0: at least one argument needed" 1>&2 - exit 1 -fi - -exec \ -env \ -EXEDESTTREE="/etc/init.d/" \ -doexe "$@" diff --git a/pkgcore/bin/ebuild-helpers/doins b/pkgcore/bin/ebuild-helpers/doins deleted file mode 100755 index 2a3fb5b..0000000 --- a/pkgcore/bin/ebuild-helpers/doins +++ /dev/null @@ -1,59 +0,0 @@ -#!/bin/bash -# Copyright 1999-2006 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 -# $Id: /var/cvsroot/gentoo-src/portage/bin/doins,v 1.7.2.2 2004/12/17 22:25:13 carpaski Exp $ - -if [ -z "${PKGCORE_BIN_PATH}" ]; then - echo "PKGCORE_BIN_PATH is unset!" - exit -1 -fi -source "${PKGCORE_BIN_PATH}/isolated-functions.sh" - -if [ $# -lt 1 ] ; then - echo "${0}: at least one argument needed" - exit 1 -fi - -if [ "${1}" == "-r" ] ; then - DOINSRECUR=y - shift -else - DOINSRECUR=n -fi -[ -z "${INSDEPTH}" ] && declare -i INSDEPTH=0 -if [ ${INSDEPTH} -gt 30 ] ; then - echo "${0}: sanity check ... 30 directories is too much :(" - exit 1 -fi - -if [ "${INSDESTTREE%${D}*}" == "" ]; then - vecho "-------------------------------------------------------" 1>&2 - vecho "You should not use \${D} with helpers." 1>&2 - vecho " --> ${INSDESTTREE}" 1>&2 - vecho "-------------------------------------------------------" 1>&2 - #exit 1 -fi - -[ ! -d "${D}${INSDESTTREE}" ] && dodir "${INSDESTTREE}" - -for x in "$@" ; do - if [ -L "$x" ] ; then - cp "$x" "${T}" - mysrc="${T}/$(/usr/bin/basename "${x}")" - elif [ -d "$x" ] ; then - if [ "${DOINSRECUR}" == "n" ] ; then - continue - fi - - mydir="${INSDESTTREE}/$(basename "${x}")" - find "${x}" -mindepth 1 -maxdepth 1 -exec \ - env \ - INSDESTTREE="${mydir}" \ - INSDEPTH=$((INSDEPTH+1)) \ - doins -r {} \; - continue - else - mysrc="${x}" - fi - install ${INSOPTIONS} "${mysrc}" "${D}${INSDESTTREE}" -done diff --git a/pkgcore/bin/ebuild-helpers/dolib b/pkgcore/bin/ebuild-helpers/dolib deleted file mode 100755 index 1a61525..0000000 --- a/pkgcore/bin/ebuild-helpers/dolib +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash -# Copyright 1999-2006 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 -# $Id: /var/cvsroot/gentoo-src/portage/bin/dolib,v 1.8.2.2 2005/01/12 02:07:15 carpaski Exp $ - -# Setup ABI cruft -LIBDIR_VAR="LIBDIR_${ABI}" -if [[ -n ${ABI} && -n ${!LIBDIR_VAR} ]] ; then - CONF_LIBDIR=${!LIBDIR_VAR} -fi -unset LIBDIR_VAR -# we need this to default to lib so that things dont break -CONF_LIBDIR=${CONF_LIBDIR:-lib} -libdir="${D}${DESTTREE}/${CONF_LIBDIR}" - - -if [[ $# -lt 1 ]] ; then - echo "$0: at least one argument needed" 1>&2 - exit 1 -fi -if [[ ! -d ${libdir} ]] ; then - install -d "${libdir}" || exit 1 -fi - -ret=0 - -for x in "$@" ; do - if [[ -e ${x} ]] ; then - if [[ ! -L ${x} ]] ; then - install ${LIBOPTIONS} "${x}" "${libdir}" - else - ln -s "$(readlink "${x}")" "${libdir}/${x##*/}" - fi - else - echo "!!! ${0##*/}: ${x} does not exist" 1>&2 - false - fi - ((ret+=$?)) -done - -exit ${ret} diff --git a/pkgcore/bin/ebuild-helpers/dolib.a b/pkgcore/bin/ebuild-helpers/dolib.a deleted file mode 100755 index c4df4a4..0000000 --- a/pkgcore/bin/ebuild-helpers/dolib.a +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -# Copyright 1999-2006 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 -# $Id: /var/cvsroot/gentoo-src/portage/bin/dolib.a,v 1.8 2004/10/10 10:07:20 carpaski Exp $ - -exec env LIBOPTIONS="-m0644" \ - dolib "$@" diff --git a/pkgcore/bin/ebuild-helpers/doman b/pkgcore/bin/ebuild-helpers/doman deleted file mode 100755 index 4c7f2bd..0000000 --- a/pkgcore/bin/ebuild-helpers/doman +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash -# Copyright 1999-2006 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 -# $Id: /var/cvsroot/gentoo-src/portage/bin/doman,v 1.13.2.2 2005/07/29 05:55:34 vapier Exp $ - -if [ -z "${PKGCORE_BIN_PATH}" ]; then - echo "PKGCORE_BIN_PATH is unset!" - exit -1 -fi -source "${PKGCORE_BIN_PATH}/isolated-functions.sh" - -if [[ $# -lt 1 ]] ; then - echo "$0: at least one argument needed" 1>&2 - exit 1 -fi - -i18n="" - -ret=0 - -for x in "$@" ; do - if [[ ${x:0:6} == "-i18n=" ]] ; then - i18n=${x:6}/ - continue - fi - if [[ ${x} == ".keep" ]] ; then - continue - fi - - suffix=${x##*.} - - if [[ ${suffix} == "gz" ]] ; then - realname=${x%.*} - suffix=${realname##*.} - vecho "QA Notice: you should let portage compress '${realname}' for you" 2>&1 - fi - - mandir=${i18n}man${suffix:0:1} - - if echo ${mandir} | egrep -q 'man[0-9n](|f|p|pm)$' -; then - if [[ -s ${x} ]] ; then - if [[ ! -d ${D}/usr/share/man/${mandir} ]] ; then - install -d "${D}/usr/share/man/${mandir}" - fi - - install -m0644 "${x}" "${D}/usr/share/man/${mandir}" - ((ret+=$?)) - elif [[ ! -e ${x} ]] ; then - vecho "doman: ${x} does not exist" 1>&2 - ((++ret)) - fi - else - vecho "doman: '${x}' is probably not a man page; skipping" 1>&2 - ((++ret)) - fi -done - -exit ${ret} diff --git a/pkgcore/bin/ebuild-helpers/domo b/pkgcore/bin/ebuild-helpers/domo deleted file mode 100755 index 8295059..0000000 --- a/pkgcore/bin/ebuild-helpers/domo +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -# Copyright 1999-2006 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 -# $Id: /var/cvsroot/gentoo-src/portage/bin/domo,v 1.6 2004/10/04 13:56:50 vapier Exp $ - -mynum=${#} -if [ ${mynum} -lt 1 ] ; then - echo "${0}: at least one argument needed" - exit 1 -fi -if [ ! -d "${D}${DESTTREE}/share/locale" ] ; then - install -d "${D}${DESTTREE}/share/locale/" -fi - -for x in "$@" ; do - if [ -e "${x}" ] ; then - mytiny="${x##*/}" - mydir="${D}${DESTTREE}/share/locale/${mytiny%.*}/LC_MESSAGES" - if [ ! -d "${mydir}" ] ; then - install -d "${mydir}" - fi - install -m0644 "${x}" "${mydir}/${MOPREFIX}.mo" - else - echo "${0}: ${x} does not exist" - fi -done diff --git a/pkgcore/bin/ebuild-helpers/donewins b/pkgcore/bin/ebuild-helpers/donewins deleted file mode 120000 index 59a0db2..0000000 --- a/pkgcore/bin/ebuild-helpers/donewins +++ /dev/null @@ -1 +0,0 @@ -newins
\ No newline at end of file diff --git a/pkgcore/bin/ebuild-helpers/dosbin b/pkgcore/bin/ebuild-helpers/dosbin deleted file mode 100755 index 30aa789..0000000 --- a/pkgcore/bin/ebuild-helpers/dosbin +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash -# Copyright 1999-2006 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 -# $Id: /var/cvsroot/gentoo-src/portage/bin/dosbin,v 1.11 2004/10/04 13:56:50 vapier Exp $ - -if [[ $# -lt 1 ]] ; then - echo "$0: at least one argument needed" 1>&2 - exit 1 -fi - -if [[ ! -d ${D}${DESTTREE}/sbin ]] ; then - install -d "${D}${DESTTREE}/sbin" || exit 2 -fi - -ret=0 - -for x in "$@" ; do - if [[ -e ${x} ]] ; then - install -m0755 -o ${PORTAGE_INST_UID:-0} -g ${PORTAGE_INST_GID:-0} "${x}" "${D}${DESTTREE}/sbin" - else - echo "!!! ${0##*/}: ${x} does not exist" 1>&2 - false - fi - ((ret+=$?)) -done - -exit ${ret} diff --git a/pkgcore/bin/ebuild-helpers/dosed b/pkgcore/bin/ebuild-helpers/dosed deleted file mode 100755 index 7422c7d..0000000 --- a/pkgcore/bin/ebuild-helpers/dosed +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -# Copyright 1999-2006 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 -# $Id: /var/cvsroot/gentoo-src/portage/bin/dosed,v 1.7 2004/10/04 13:56:50 vapier Exp $ - -mysed="s:${D}::g" - -for x in "$@" ; do - y="${D}${x}" - if [ -e "${y}" ] ; then - if [ -f "${y}" ] ; then - mysrc="${T}/${y##*/}" - cp "${y}" "${mysrc}" - sed -e "${mysed}" "${mysrc}" > "${y}" - else - echo "${y} is not a regular file!" - exit 1 - fi - else - mysed="${x}" - fi -done diff --git a/pkgcore/bin/ebuild-helpers/dosym b/pkgcore/bin/ebuild-helpers/dosym deleted file mode 100755 index e0af15e..0000000 --- a/pkgcore/bin/ebuild-helpers/dosym +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -# Copyright 1999-2006 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 -# $Id: /var/cvsroot/gentoo-src/portage/bin/dosym,v 1.7 2004/10/04 13:56:50 vapier Exp $ - -if [[ $# -ne 2 ]] ; then - echo "$0: two arguments needed" 1>&2 - exit 1 -fi - -destdir=${2%/*} -[[ ! -d ${D}${destdir} ]] && dodir "${destdir}" - -exec ln -snf "$1" "${D}$2" diff --git a/pkgcore/bin/ebuild-helpers/emake b/pkgcore/bin/ebuild-helpers/emake deleted file mode 100755 index d9f548f..0000000 --- a/pkgcore/bin/ebuild-helpers/emake +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -# Copyright 1999-2005 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 -# $Id: emake 1912 2005-08-25 03:54:42Z ferringb $ -# -# emake: Supplies some default parameters to GNU make. At the moment the -# only parameter supplied is -jN, where N is a number of -# parallel processes that should be ideal for the running host -# (e.g. on a single-CPU machine, N=2). The MAKEOPTS variable -# is set in /etc/make.globals. We don't source -# /etc/make.globals here because emake is only called from an -# ebuild. - -exec ${MAKE:-make} ${MAKEOPTS} ${EXTRA_EMAKE} "$@" diff --git a/pkgcore/bin/ebuild-helpers/fowners b/pkgcore/bin/ebuild-helpers/fowners deleted file mode 100755 index 99f0685..0000000 --- a/pkgcore/bin/ebuild-helpers/fowners +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -# Copyright 1999-2005 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 -# $Id: fowners 1912 2005-08-25 03:54:42Z ferringb $ - -slash=/ -exec chown "${@/#${slash}/${D}${slash}}" diff --git a/pkgcore/bin/ebuild-helpers/fperms b/pkgcore/bin/ebuild-helpers/fperms deleted file mode 100755 index 383894e..0000000 --- a/pkgcore/bin/ebuild-helpers/fperms +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -# Copyright 1999-2005 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 -# $Id: fperms 1912 2005-08-25 03:54:42Z ferringb $ - -slash=/ -exec chmod "${@/#${slash}/${D}${slash}}" diff --git a/pkgcore/bin/ebuild-helpers/newbin b/pkgcore/bin/ebuild-helpers/newbin deleted file mode 100755 index ba7852d..0000000 --- a/pkgcore/bin/ebuild-helpers/newbin +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -# Copyright 1999-2004 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 -# $Id: newbin 1912 2005-08-25 03:54:42Z ferringb $ - -if [ -z "${T}" ] || [ -z "${2}" ] ; then - echo "Nothing defined to do." - exit 1 -fi - -rm -rf "${T}/${2}" -cp "${1}" "${T}/${2}" -dobin "${T}/${2}" diff --git a/pkgcore/bin/ebuild-helpers/newconfd b/pkgcore/bin/ebuild-helpers/newconfd deleted file mode 100755 index e9f2aa5..0000000 --- a/pkgcore/bin/ebuild-helpers/newconfd +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -# Copyright 1999-2005 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 -# $Id: newconfd 1912 2005-08-25 03:54:42Z ferringb $ - -if [ -z "${T}" ] || [ -z "${2}" ] ; then - echo "$0: nothing defined to do" 1>&2 - exit 1 -fi - -rm -rf "${T}/${2}" -cp "${1}" "${T}/${2}" -exec doconfd "${T}/${2}" diff --git a/pkgcore/bin/ebuild-helpers/newdoc b/pkgcore/bin/ebuild-helpers/newdoc deleted file mode 100755 index bc56d73..0000000 --- a/pkgcore/bin/ebuild-helpers/newdoc +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -# Copyright 1999-2005 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 -# $Id: newdoc 1912 2005-08-25 03:54:42Z ferringb $ - -if [ -z "${T}" ] || [ -z "${2}" ] ; then - echo "newdoc: Nothing defined to do" 1>&2 - exit 1 -fi - -rm -rf "${T}/${2}" -cp "${1}" "${T}/${2}" -exec dodoc "${T}/${2}" diff --git a/pkgcore/bin/ebuild-helpers/newenvd b/pkgcore/bin/ebuild-helpers/newenvd deleted file mode 100755 index 68cf65c..0000000 --- a/pkgcore/bin/ebuild-helpers/newenvd +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -# Copyright 1999-2005 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 -# $Id: newenvd 1912 2005-08-25 03:54:42Z ferringb $ - -if [ -z "${T}" ] || [ -z "${2}" ] ; then - echo "$0: nothing defined to do" 1>&2 - exit 1 -fi - -rm -rf "${T}/${2}" -cp "${1}" "${T}/${2}" -exec doenvd "${T}/${2}" diff --git a/pkgcore/bin/ebuild-helpers/newexe b/pkgcore/bin/ebuild-helpers/newexe deleted file mode 100755 index 4769694..0000000 --- a/pkgcore/bin/ebuild-helpers/newexe +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -# Copyright 1999-2004 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 -# $Id: newexe 1912 2005-08-25 03:54:42Z ferringb $ - -if [ -z "${T}" ] || [ -z "${2}" ] ; then - echo "Nothing defined to do." - exit 1 -fi - -rm -rf "${T}/${2}" -cp "${1}" "${T}/${2}" -doexe "${T}/${2}" diff --git a/pkgcore/bin/ebuild-helpers/newinitd b/pkgcore/bin/ebuild-helpers/newinitd deleted file mode 100755 index f461bba..0000000 --- a/pkgcore/bin/ebuild-helpers/newinitd +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -# Copyright 1999-2005 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 -# $Id: newinitd 1912 2005-08-25 03:54:42Z ferringb $ - -if [ -z "${T}" ] || [ -z "${2}" ] ; then - echo "$0: nothing defined to do" 1>&2 - exit 1 -fi - -rm -rf "${T}/${2}" -cp "${1}" "${T}/${2}" -exec doinitd "${T}/${2}" diff --git a/pkgcore/bin/ebuild-helpers/newins b/pkgcore/bin/ebuild-helpers/newins deleted file mode 100755 index bb89feb..0000000 --- a/pkgcore/bin/ebuild-helpers/newins +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -# Copyright 1999-2004 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 -# $Id: newins 1912 2005-08-25 03:54:42Z ferringb $ - -if [ -z "${T}" ] || [ -z "${2}" ] ; then - echo "Error: Nothing defined to do." - exit 1 -fi - -rm -rf "${T}/${2}" -cp "${1}" "${T}/${2}" -doins "${T}/${2}" diff --git a/pkgcore/bin/ebuild-helpers/newlib.a b/pkgcore/bin/ebuild-helpers/newlib.a deleted file mode 100755 index ac4b035..0000000 --- a/pkgcore/bin/ebuild-helpers/newlib.a +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -# Copyright 1999-2004 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 -# $Id: newlib.a 1912 2005-08-25 03:54:42Z ferringb $ - -if [ -z "${T}" ] || [ -z "${2}" ] ; then - echo "Error: Nothing defined to do." - exit 1 -fi - -rm -rf "${T}/${2}" -cp "${1}" "${T}/${2}" -dolib.a "${T}/${2}" diff --git a/pkgcore/bin/ebuild-helpers/newman b/pkgcore/bin/ebuild-helpers/newman deleted file mode 100755 index 0081851..0000000 --- a/pkgcore/bin/ebuild-helpers/newman +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -# Copyright 1999-2004 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 -# $Id: newman 1912 2005-08-25 03:54:42Z ferringb $ - -if [ -z "${T}" ] || [ -z "${2}" ] ; then - echo "newman: Nothing defined to do" 1>&2 - exit 1 -fi - -rm -rf "${T}/${2}" -cp "${1}" "${T}/${2}" -exec doman "${T}/${2}" diff --git a/pkgcore/bin/ebuild-helpers/newsbin b/pkgcore/bin/ebuild-helpers/newsbin deleted file mode 100755 index ab9f397..0000000 --- a/pkgcore/bin/ebuild-helpers/newsbin +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -# Copyright 1999-2004 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 -# $Id: newsbin 1912 2005-08-25 03:54:42Z ferringb $ - -if [ -z "${T}" ] || [ -z "${2}" ] ; then - echo "Nothing defined to do." - exit 1 -fi - -rm -rf "${T}/${2}" -cp "${1}" "${T}/${2}" -dosbin "${T}/${2}" diff --git a/pkgcore/bin/ebuild-helpers/prepall b/pkgcore/bin/ebuild-helpers/prepall deleted file mode 100755 index 1f1f458..0000000 --- a/pkgcore/bin/ebuild-helpers/prepall +++ /dev/null @@ -1,86 +0,0 @@ -#!/bin/bash -# Copyright 1999-2005 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 -# $Id: prepall 2394 2005-12-17 17:23:07Z vapier $ - -prepallman -prepallinfo -prepallstrip - -# this should help to ensure that all (most?) shared libraries are executable -# and that all libtool scripts / static libraries are not executable -for i in "${D}"opt/*/lib{,32,64} \ - "${D}"lib{,32,64} \ - "${D}"usr/lib{,32,64} \ - "${D}"usr/X11R6/lib{,32,64} ; do - [[ ! -d ${i} ]] && continue - - for j in "${i}"/*.so.* "${i}"/*.so ; do - [[ ! -e ${j} ]] && continue - [[ -L ${j} ]] && continue - [[ -x ${j} ]] && continue - echo "making executable: /${j/${D}/}" - chmod +x "${j}" - done - - for j in "${i}"/*.a "${i}"/*.la ; do - [[ ! -e ${j} ]] && continue - [[ -L ${j} ]] && continue - [[ ! -x ${j} ]] && continue - echo "removing executable bit: /${j/${D}/}" - chmod -x "${j}" - done -done - -# When installing static libraries into /usr/lib and shared libraries into -# /lib, we have to make sure we have a linker script in /usr/lib along side -# the static library, or gcc will utilize the static lib when linking :(. -# http://bugs.gentoo.org/4411 -for a in "${D}"usr/lib*/*.a ; do - s=${a%.a}.so - if [[ ! -e ${s} ]] ; then - s=${s%usr/*}${s##*/usr/} - if [[ -e ${s} ]] ; then - echo -e "\aQA Notice: missing gen_usr_ldscript for ${s##*/}\a" - sleep 1 - fi - fi -done - -# Make sure people don't store libtool files or static libs in /lib -f=$(ls "${D}"lib*/*.{a,la} 2>/dev/null) -if [[ -n ${f} ]] ; then - echo -e "\n\aQA Notice: excessive files found in the / partition\a" - echo "${f}" - sleep 1 -fi - -# Verify that the libtool files don't contain bogus $D entries. -for a in "${D}"usr/lib*/*.la ; do - s=${a##*/} - if grep -qs "${D}" "${a}" ; then - echo -e "\n\aQA Notice: ${s} appears to contain PORTAGE_TMPDIR paths\a" - sleep 1 - fi -done - -if type -p scanelf > /dev/null ; then - -# Run some sanity checks on shared libraries -for d in "${D}"lib* "${D}"usr/lib* ; do - f=$(scanelf -ByF '%S %p' "${d}"/lib*.so* | gawk '$2 == "" { print }') - if [[ -n ${f} ]] ; then - echo -e "\n\aQA Notice: the following shared libraries lack a SONAME\a" - echo "${f}" - sleep 1 - fi - - f=$(scanelf -ByF '%n %p' "${d}"/lib*.so* | gawk '$2 == "" { print }') - if [[ -n ${f} ]] ; then - echo -e "\n\aQA Notice: the following shared libraries lack NEEDED entries\a" - echo "${f}" - sleep 1 - fi -done - -fi diff --git a/pkgcore/bin/ebuild-helpers/prepalldocs b/pkgcore/bin/ebuild-helpers/prepalldocs deleted file mode 100755 index e71c6e4..0000000 --- a/pkgcore/bin/ebuild-helpers/prepalldocs +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/bash -# Copyright 1999-2005 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 -# $Id: prepalldocs 1912 2005-08-25 03:54:42Z ferringb $ - -dir="${D}usr/share/doc" - -[ ! -d "${dir}" ] && exit 0 - -z=$(find "${dir}" \ - '(' -type f -or -type l ')' \ - -not -name '*.gz' \ - -not -name '*.bz2' \ - -not -name '*.Z' \ - -not -name '*.js' \ - 2>/dev/null) - -[ -z "${z}" ] && exit 0 - -PORTAGE_COMPRESS=${PORTAGE_COMPRESS:-gzip} -PORTAGE_COMPRESS_FLAGS=${PORTAGE_COMPRESS_FLAGS:--9} -if [ -z "${PORTAGE_COMPRESS_SUFFIX}" ] ; then - case ${PORTAGE_COMPRESS} in - gzip) suffix="gz";; - bzip2) suffix="bz2";; - *) echo "prepalldocs error: please set PORTAGE_COMPRESS_SUFFIX in make.conf" 1>&2 - exit 1;; - esac -fi - -echo "doc: ${PORTAGE_COMPRESS} ${PORTAGE_COMPRESS_FLAGS}" -for y in ${z} ; do - if [ -L "${y}" ] ; then - # Symlink ... - mylink=${y} - linkto=$(readlink "${y}") - - if [ "${linkto##*.}" != "${suffix}" ] ; then - linkto="${linkto}.${suffix}" - fi - if [ "${mylink##*.}" != "${suffix}" ] ; then - mylink="${mylink}.${suffix}" - fi - - echo " link fixed ${mylink##*/}" - ln -snf "${linkto}" "${mylink}" - if [ "${y}" != "${mylink}" ] ; then - echo " link removed ${y##*/}" - rm -f "${y}" - fi - else - if [ "${y##*.}" != "${suffix}" ] ; then - echo " compressing ${y##*/}" - "${PORTAGE_COMPRESS}" ${PORTAGE_COMPRESS_FLAGS} -f "${y}" - fi - fi -done diff --git a/pkgcore/bin/ebuild-helpers/prepallinfo b/pkgcore/bin/ebuild-helpers/prepallinfo deleted file mode 100755 index 220391e..0000000 --- a/pkgcore/bin/ebuild-helpers/prepallinfo +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -# Copyright 1999-2005 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 -# $Id: prepallinfo 1912 2005-08-25 03:54:42Z ferringb $ - -[ ! -d "${D}usr/share/info" ] && exit 0 - -exec prepinfo diff --git a/pkgcore/bin/ebuild-helpers/prepallman b/pkgcore/bin/ebuild-helpers/prepallman deleted file mode 100755 index 77b570a..0000000 --- a/pkgcore/bin/ebuild-helpers/prepallman +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash -# Copyright 1999-2005 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 -# $Id: prepallman 1912 2005-08-25 03:54:42Z ferringb $ - -for x in $(find "${D}" -name man -type d -printf '%P\n') ; do - prepman ${x%/man} - export prepallman_banner=no -done diff --git a/pkgcore/bin/ebuild-helpers/prepallstrip b/pkgcore/bin/ebuild-helpers/prepallstrip deleted file mode 100755 index e55e111..0000000 --- a/pkgcore/bin/ebuild-helpers/prepallstrip +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -# Copyright 1999-2005 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 -# $Id: prepallstrip 1912 2005-08-25 03:54:42Z ferringb $ - -if [ "${FEATURES//*nostrip*/true}" == "true" ] || [ "${RESTRICT//*nostrip*/true}" == "true" ] ; then - exit 0 -fi - -exec prepstrip "${D}" diff --git a/pkgcore/bin/ebuild-helpers/prepinfo b/pkgcore/bin/ebuild-helpers/prepinfo deleted file mode 100755 index 85af086..0000000 --- a/pkgcore/bin/ebuild-helpers/prepinfo +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash -# Copyright 1999-2005 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 -# $Id: prepinfo 1912 2005-08-25 03:54:42Z ferringb $ - -if [ -z "$1" ] ; then - z="${D}usr/share/info" -else - if [ -d "${D}$1/share/info" ] ; then - z="${D}$1/share/info" - else - z="${D}$1/info" - fi -fi - -[ ! -d "${z}" ] && exit 0 - -rm -f "${z}"/dir{,.old}{,.info{,.gz,.bz2,.Z}} - -PORTAGE_COMPRESS=${PORTAGE_COMPRESS:-gzip} -PORTAGE_COMPRESS_FLAGS=${PORTAGE_COMPRESS_FLAGS:--9} -if [ -z "${PORTAGE_COMPRESS_SUFFIX}" ] ; then - case ${PORTAGE_COMPRESS} in - gzip) suffix="gz";; - bzip2) suffix="bz2";; - *) echo "prepinfo: error fixing links: please set PORTAGE_COMPRESS_SUFFIX in make.conf" 1>&2 - exit 1;; - esac -fi - -echo "info: ${PORTAGE_COMPRESS} ${PORTAGE_COMPRESS_FLAGS}" - -for x in `find "${z}"/ \( -type f -or -type l \) -maxdepth 1 -mindepth 1 2>/dev/null` ; do - if [ -L "${x}" ] ; then - # Symlink ... - mylink=${x} - linkto=$(readlink "${x}") - - if [ "${linkto##*.}" != "${suffix}" ] ; then - linkto="${linkto}.${suffix}" - fi - if [ "${mylink##*.}" != "${suffix}" ] ; then - mylink="${mylink}.${suffix}" - fi - - echo "fixing GNU info symlink: ${mylink##*/}" - ln -snf "${linkto}" "${mylink}" - if [ "${x}" != "${mylink}" ] ; then - echo "removing old symlink: ${x##*/}" - rm -f "${x}" - fi - else - if [ "${x##*.}" != "${suffix}" ] ; then - echo "compressing GNU info page: ${x##*/}" - "${PORTAGE_COMPRESS}" ${PORTAGE_COMPRESS_FLAGS} -f "${x}" - fi - fi -done diff --git a/pkgcore/bin/ebuild-helpers/preplib b/pkgcore/bin/ebuild-helpers/preplib deleted file mode 100755 index c6ea5c4..0000000 --- a/pkgcore/bin/ebuild-helpers/preplib +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -# Copyright 1999-2004 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 -# $Id: preplib 1912 2005-08-25 03:54:42Z ferringb $ - -LIBDIR_VAR="LIBDIR_${ABI}" -if [ -n "${ABI}" -a -n "${!LIBDIR_VAR}" ]; then - CONF_LIBDIR="${!LIBDIR_VAR}" -fi -unset LIBDIR_VAR - -if [ -z "${CONF_LIBDIR}" ]; then - # we need this to default to lib so that things dont break - CONF_LIBDIR="lib" -fi - -if [ -z "$1" ] ; then - z="${D}usr/${CONF_LIBDIR}" -else - z="${D}$1/${CONF_LIBDIR}" -fi - -if [ -d "${z}" ] ; then - ldconfig -n -N "${z}" -fi diff --git a/pkgcore/bin/ebuild-helpers/prepman b/pkgcore/bin/ebuild-helpers/prepman deleted file mode 100755 index 0fd16f5..0000000 --- a/pkgcore/bin/ebuild-helpers/prepman +++ /dev/null @@ -1,61 +0,0 @@ -#!/bin/bash -# Copyright 1999-2005 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 -# $Id: prepman 1912 2005-08-25 03:54:42Z ferringb $ - -if [ -z "$1" ] ; then - z="${D}usr/share/man" -else - z="${D}$1/man" -fi - -[ ! -d "${z}" ] && exit 0 - -PORTAGE_COMPRESS=${PORTAGE_COMPRESS:-gzip} -PORTAGE_COMPRESS_FLAGS=${PORTAGE_COMPRESS_FLAGS:--9} -if [ -z "${PORTAGE_COMPRESS_SUFFIX}" ] ; then - case ${PORTAGE_COMPRESS} in - gzip) suffix="gz";; - bzip2) suffix="bz2";; - *) echo "prepman error: please set PORTAGE_COMPRESS_SUFFIX in make.conf" 1>&2 - exit 1;; - esac -fi - -if [ -z "${prepallman_banner}" ] ; then - echo "man: ${PORTAGE_COMPRESS} ${PORTAGE_COMPRESS_FLAGS}" -fi - -for x in `find "${z}"/ -type d 2>/dev/null` ; do - for y in `find "${x}"/ \( -type f -or -type l \) ! -name '.keep' -maxdepth 1 -mindepth 1 2>/dev/null` ; do - if [ -L "${y}" ] ; then - # Symlink ... - mylink=${y} - linkto=$(readlink "${y}") - - # Do NOT change links to directories - if [ -d "${z}/${linkto}" ] ; then - continue - fi - - if [ "${linkto##*.}" != "${suffix}" ] ; then - linkto="${linkto}.${suffix}" - fi - if [ "${mylink##*.}" != "${suffix}" ] ; then - mylink="${mylink}.${suffix}" - fi - - echo " link fixed ${mylink##*/}" - ln -snf "${linkto}" "${mylink}" - if [ "${y}" != "${mylink}" ] ; then - echo " link removed ${y##*/}" - rm -f "${y}" - fi - else - if [ "${y##*.}" != "${suffix}" ] && [ ! -d "${y}" ] ; then - echo " compressing ${y##*/}" - "${PORTAGE_COMPRESS}" ${PORTAGE_COMPRESS_FLAGS} -f "${y}" - fi - fi - done -done diff --git a/pkgcore/bin/ebuild-helpers/prepstrip b/pkgcore/bin/ebuild-helpers/prepstrip deleted file mode 100755 index 2e03cb9..0000000 --- a/pkgcore/bin/ebuild-helpers/prepstrip +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/bash -# Copyright 1999-2005 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 -# $Id: prepstrip 2228 2005-11-01 01:35:23Z vapier $ - -if [ "${FEATURES//*nostrip*/true}" == "true" ] || [ "${RESTRICT//*nostrip*/true}" == "true" ] ; then - echo "nostrip" - STRIP="/bin/false" - PORTAGE_STRIP_FLAGS="" -else - STRIP=${STRIP:-${CHOST}-strip} - type -p -- ${STRIP} > /dev/null || STRIP=strip - PORTAGE_STRIP_FLAGS=${PORTAGE_STRIP_FLAGS:---strip-unneeded} -fi - -banner=1 -retval=0 - -for x in "$@" ; do - if [ -d "${x}" ]; then - # We only want files. So make a pass for each directory and call again. - find "${x}" -type f \( -perm -0100 -or -perm -0010 -or -perm -0001 -or -name '*.so' -or -name '*.so.*' \) -print0 | - $XARGS -0 -n500 prepstrip - else - if [ ${banner} -eq 1 ] ; then - echo "strip: ${STRIP} ${PORTAGE_STRIP_FLAGS}" - banner=0 - fi - - f=$(file "${x}") || continue - [ -z "${f}" ] && continue - - if [ -z "${f/*current ar archive*/}" ]; then - echo " ${x:${#D}:${#x}}" - ${STRIP} -g "${x}" - fi - if [ -z "${f/*SB executable*/}" ]; then - echo " ${x:${#D}:${#x}}" - ${STRIP} "${x}" - fi - if [ -z "${f/*SB shared object*/}" ]; then - echo " ${x:${#D}:${#x}}" - ${STRIP} ${PORTAGE_STRIP_FLAGS} "${x}" - fi - fi -done - -exit ${retval} diff --git a/pkgcore/binpkg/__init__.py b/pkgcore/binpkg/__init__.py deleted file mode 100644 index 23646cf..0000000 --- a/pkgcore/binpkg/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -""" -gentoo binpkg support -""" diff --git a/pkgcore/binpkg/repo_ops.py b/pkgcore/binpkg/repo_ops.py deleted file mode 100644 index bae5e0e..0000000 --- a/pkgcore/binpkg/repo_ops.py +++ /dev/null @@ -1,99 +0,0 @@ -# Copyright: 2006 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -import os, errno - -from pkgcore.interfaces import repo as repo_interfaces -from pkgcore.fs import tar -from pkgcore.binpkg import xpak -from pkgcore.ebuild.conditionals import stringify_boolean - -from snakeoil import osutils -from pkgcore.util.bzip2 import compress -from snakeoil.osutils import join as pjoin -from snakeoil.demandload import demandload -demandload(globals(), "pkgcore.log:logger") - -def discern_loc(base, pkg): - return pjoin(base, pkg.category, - "%s-%s.tbz2" % (pkg.package, pkg.fullver)) - - -_metadata_rewrites = { - "depends":"DEPEND", "rdepends":"RDEPEND", "post_rdepends":"PDEPEND", - "use":"USE", "eapi":"EAPI", "CONTENTS":"contents", "provides":"PROVIDE"} - -def generate_attr_dict(pkg): - d = {} - for k in pkg.tracked_attributes: - if k == "contents": - continue - v = getattr(pkg, k) - if k == 'environment': - d['environment.bz2'] = compress(v.get_fileobj().read()) - continue - if k == 'provides': - versionless_provides = lambda b: b.key - s = stringify_boolean(v, func=versionless_provides) - elif not isinstance(v, basestring): - try: - s = ' '.join(v) - except TypeError: - s = str(v) - else: - s = v - d[_metadata_rewrites.get(k, k.upper())] = s - d["%s-%s.ebuild" % (pkg.package, pkg.fullver)] = \ - pkg.ebuild.get_fileobj().read() - return d - - -class install(repo_interfaces.nonlivefs_install): - - def modify_repo(self): - if self.observer is None: - end = start = lambda x:None - else: - start = self.observer.phase_start - end = self.observer.phase_end - pkg = self.new_pkg - final_path = discern_loc(self.repo.base, pkg) - tmp_path = pjoin(os.path.dirname(final_path), - ".tmp.%i.%s" % (os.getpid(), os.path.basename(final_path))) - - if not osutils.ensure_dirs(os.path.dirname(tmp_path), mode=0755): - raise repo_interfaces.Failure("failed creating directory %r" % - os.path.dirname(tmp_path)) - try: - start("generating tarball: %s" % tmp_path) - tar.write_set(pkg.contents, tmp_path, compressor='bz2') - end("tarball created") - start("writing Xpak") - # ok... got a tarball. now add xpak. - x = xpak.Xpak.write_xpak(tmp_path, generate_attr_dict(pkg)) - end("wrote Xpak") - # ok... we tagged the xpak on. - os.chmod(tmp_path, 0644) - os.rename(tmp_path, final_path) - except: - try: - os.unlink(tmp_path) - except (IOError, OSError), e: - if e.errno != errno.ENOENT: - logger.warn("failed removing %r: %r" % (tmp_path, e)) - raise - return True - - -class uninstall(repo_interfaces.nonlivefs_uninstall): - - def modify_repo(self): - os.unlink(discern_loc(self.repo.base, self.old_pkg)) - return True - - -class replace(install, uninstall, repo_interfaces.nonlivefs_replace): - - def modify_repo(self): - uninstall.modify_repo(self) - install.modify_repo(self) diff --git a/pkgcore/binpkg/repository.py b/pkgcore/binpkg/repository.py deleted file mode 100644 index a0804aa..0000000 --- a/pkgcore/binpkg/repository.py +++ /dev/null @@ -1,298 +0,0 @@ -# Copyright: 2005-2007 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -""" -binpkg ebuild repository -""" - -import os, stat - -from pkgcore.repository import prototype, errors -from pkgcore.merge import triggers -from pkgcore.plugin import get_plugin -from pkgcore.ebuild.ebuild_built import pkg_uses_default_preinst -from pkgcore.config import ConfigHint -#needed to grab the PN -from pkgcore.ebuild.cpv import CPV as cpv - -from snakeoil.currying import partial -from snakeoil.mappings import DictMixin -from snakeoil.osutils import listdir_dirs, listdir_files -from snakeoil.osutils import join as pjoin - -from snakeoil.demandload import demandload -demandload(globals(), - "pkgcore.merge:engine", - "pkgcore.fs.livefs:scan", - "pkgcore.interfaces.data_source:local_source", - "pkgcore.fs.ops:offset_rewriter", - "pkgcore.interfaces.data_source:data_source", - "pkgcore.repository:wrapper", - "pkgcore.package.mutated:MutatedPkg", - "pkgcore.ebuild:ebd", - "pkgcore.binpkg:repo_ops", - "errno", - "pkgcore.fs.tar:generate_contents", - "pkgcore.binpkg.xpak:Xpak", - "pkgcore.util.bzip2:decompress", -) - - -class force_unpacking(triggers.base): - - required_csets = ('install',) - _hooks = ('sanity_check',) - _priority = 5 - _label = 'forced decompression' - _engine_type = triggers.INSTALLING_MODES - - def __init__(self, format_op): - self.format_op = format_op - - def trigger(self, engine, cset): - op = self.format_op - op.setup_workdir() - merge_contents = get_plugin("fs_ops.merge_contents") - merge_cset = cset - if engine.offset != '/': - merge_cset = cset.change_offset(engine.offset, '/') - merge_contents(merge_cset, offset=op.env["D"]) - - # ok. they're on disk. - # now to avoid going back to the binpkg, we rewrite - # the data_source for files to the on disk location. - # we can update in place also, since we're not changing the mapping. - - # this rewrites the data_source to the ${D} loc. - d = op.env["D"] - fi = (x.change_attributes(data_source=local_source( - pjoin(d, x.location.lstrip('/')))) - for x in merge_cset.iterfiles()) - - if engine.offset: - # we're using merge_cset above, which has the final offset loc - # pruned; this is required for the merge, however, we're updating - # the cset so we have to insert the final offset back in. - # wrap the iter, iow. - fi = offset_rewriter(engine.offset, fi) - - # we *probably* should change the csets class at some point - # since it no longer needs to be tar, but that's for another day. - cset.update(fi) - - -def wrap_factory(klass, *args, **kwds): - - class new_factory(klass): - - def _add_format_triggers(self, pkg, op_inst, format_op_inst, - engine_inst): - if engine.UNINSTALL_MODE != engine_inst.mode and \ - pkg == engine_inst.new and \ - pkg.repo is engine_inst.new.repo and \ - not pkg_uses_default_preinst(pkg): - t = force_unpacking(op_inst.install_op) - t.register(engine_inst) - - klass._add_format_triggers( - self, pkg, op_inst, format_op_inst, engine_inst) - - def scan_contents(self, location): - return scan(location, offset=location) - - return new_factory(*args, **kwds) - - -class StackedXpakDict(DictMixin): - __slots__ = ("_xpak", "_parent", "_pkg", "contents", - "_wipes") - - _metadata_rewrites = { - "depends":"DEPEND", "rdepends":"RDEPEND", "post_rdepends":"PDEPEND", - "provides":"PROVIDE", "use":"USE", "eapi":"EAPI", - "CONTENTS":"contents"} - - def __init__(self, parent, pkg): - self._pkg = pkg - self._parent = parent - self._wipes = [] - - def __getattr__(self, attr): - if attr == "_xpak": - data = Xpak(self._parent._get_path(self._pkg)) - object.__setattr__(self, attr, data) - return data - raise AttributeError(self, attr) - - def __getitem__(self, key): - key = self._metadata_rewrites.get(key, key) - if key in self._wipes: - raise KeyError(self, key) - if key == "contents": - data = generate_contents(self._parent._get_path(self._pkg)) - object.__setattr__(self, "contents", data) - elif key == "environment": - data = self._xpak.get("environment.bz2") - if data is None: - data = data_source(self._xpak.get("environment"), - mutable=True) - if data is None: - raise KeyError( - "environment.bz2 not found in xpak segment, " - "malformed binpkg?") - else: - data = data_source(decompress(data), mutable=True) - elif key == "ebuild": - data = self._xpak.get("%s-%s.ebuild" % - (self._pkg.package, self._pkg.fullver), "") - data = data_source(data) - else: - try: - data = self._xpak[key] - except KeyError: - data = '' - return data - - def __delitem__(self, key): - if key in ("contents", "environment"): - if key in self._wipes: - raise KeyError(self, key) - self._wipes.append(key) - else: - del self._xpak[key] - - def __setitem__(self, key, val): - if key in ("contents", "environment"): - setattr(self, key, val) - self._wipes = [x for x in self._wipes if x != key] - else: - self._xpak[key] = val - return val - - def iterkeys(self): - for k in self._xpak: - yield k - for k in ("environment", "contents"): - if self.get(k) is not None: - yield k - - -class tree(prototype.tree): - - format_magic = "ebuild_built" - - # yes, the period is required. no, do not try and remove it - # (harring says it stays) - extension = ".tbz2" - - configured = False - configurables = ("settings",) - - pkgcore_config_type = ConfigHint({'location':'str', - 'repo_id':'str'}, typename='repo') - - def __init__(self, location, repo_id=None): - super(tree, self).__init__() - self.base = location - if repo_id is None: - repo_id = location - self.repo_id = repo_id - self._versions_tmp_cache = {} - try: - st = os.lstat(self.base) - if not stat.S_ISDIR(st.st_mode): - raise errors.InitializationError( - "base not a dir: %s" % self.base) - elif not st.st_mode & (os.X_OK|os.R_OK): - raise errors.InitializationError( - "base lacks read/executable: %s" % self.base) - - except OSError: - raise errors.InitializationError( - "lstat failed on base %s" % self.base) - - self.package_class = wrap_factory( - get_plugin("format." + self.format_magic), self) - - def _get_categories(self, *optional_category): - # return if optional_category is passed... cause it's not yet supported - if optional_category: - return {} - try: - return tuple( - x for x in listdir_dirs(self.base) - if x.lower() != "all") - except (OSError, IOError), e: - raise KeyError("failed fetching categories: %s" % str(e)) - - def _get_packages(self, category): - cpath = pjoin(self.base, category.lstrip(os.path.sep)) - l = set() - d = {} - lext = len(self.extension) - try: - for x in listdir_files(cpath): - # don't use lstat; symlinks may exist - if (x.endswith(".lockfile") - or not x[-lext:].lower() == self.extension): - continue - x = cpv(category+"/"+x[:-lext]) - l.add(x.package) - d.setdefault((category, x.package), []).append(x.fullver) - except (OSError, IOError), e: - raise KeyError("failed fetching packages for category %s: %s" % \ - (pjoin(self.base, category.lstrip(os.path.sep)), str(e))) - - self._versions_tmp_cache.update(d) - return tuple(l) - - def _get_versions(self, catpkg): - return tuple(self._versions_tmp_cache.pop(catpkg)) - - def _get_path(self, pkg): - s = "%s-%s" % (pkg.package, pkg.fullver) - return pjoin(self.base, pkg.category, s+".tbz2") - - _get_ebuild_path = _get_path - - def _get_metadata(self, pkg): - return StackedXpakDict(self, pkg) - - def notify_remove_package(self, pkg): - prototype.tree.notify_remove_package(self, pkg) - try: - os.rmdir(pjoin(self.base, pkg.category)) - except OSError, oe: - if oe.errno != errno.ENOTEMPTY: - raise - del oe - - def _install(self, pkg, *a, **kw): - return repo_ops.install(self, pkg, *a, **kw) - - def _uninstall(self, pkg, *a, **kw): - return repo_ops.uninstall(self, pkg, *a, **kw) - - def _replace(self, oldpkg, newpkg, *a, **kw): - return repo_ops.replace(self, oldpkg, newpkg, *a, **kw) - - -class ConfiguredBinpkgTree(wrapper.tree): - - format_magic = "ebuild_built" - configured = True - - def __init__(self, repo, domain_settings): - # rebind to ourselves basically. - def package_class(pkg): - return MutatedPkg(pkg, - {"build":partial(self._generate_build_op, pkg)}) - wrapper.tree.__init__(self, repo, package_class=package_class) - self.domain_settings = domain_settings - - def _generate_build_op(self, pkg, **kwargs): - kwargs["initial_env"] = self.domain_settings - kwargs["env_data_source"] = pkg.environment - return ebd.binpkg_buildable(pkg, **kwargs) - -tree.configure = ConfiguredBinpkgTree diff --git a/pkgcore/binpkg/xpak.py b/pkgcore/binpkg/xpak.py deleted file mode 100644 index 1d64985..0000000 --- a/pkgcore/binpkg/xpak.py +++ /dev/null @@ -1,263 +0,0 @@ -# Copyright: 2006-2007 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -""" -XPAK container support -""" - -import struct -from snakeoil.mappings import OrderedDict -from snakeoil.demandload import demandload -demandload(globals(), "os", "errno") - -# -# format is: -# XPAKPACKIIIIDDDD[index][data]XPAKSTOPOOOOSTOP -# first; all ints/longs are big endian -# meanwhile, 8 byte format magic -# 4 bytes of index len, -# 4 bytes of data len -# index items: 4 bytes (len of the key name), then that length of key data -# finally, 2 longs; relative offset from data block start, length of the data -# repeats till index is full processed -# for data, just a big blob; offsets into it are determined via the index -# table. -# finally, trailing magic, 4 bytes (positive) of the # of bytes to seek to -# reach the end of the magic, and 'STOP'. offset is relative to EOS for Xpak -# - -class MalformedXpak(Exception): - def __init__(self, msg): - Exception.__init__(self, "xpak as malformed: %s" % (msg,)) - self.msg = msg - - -class Xpak(object): - __slots__ = ("_source", "_source_is_path", "xpak_start", "_keys_dict") - trailer_size = 16 - trailer_parser = ">8sL4s" - trailer_pre_magic = "XPAKSTOP" - trailer_post_magic = "STOP" - - header_size = 16 - header_parser = ">8sLL" - header_pre_magic = "XPAKPACK" - - - def __init__(self, source): - self._source_is_path = isinstance(source, basestring) - self._source = source - self.xpak_start = None - # _keys_dict becomes an ordereddict after _load_offsets; reason for - # it is so that reads are serialized. - - def __getattr__(self, attr): - if attr == "_keys_dict": - self._load_offsets() - return object.__getattribute__(self, attr) - raise AttributeError(self, attr) - - @property - def _fd(self): - # we do this annoying little dance to avoid having a couple - # hundred fds open if they're accessing a lot of binpkgs - if self._source_is_path: - return open(self._source, "r") - return self._source - - @classmethod - def write_xpak(cls, target_source, data): - """ - write an xpak dict to disk; overwriting an xpak if it exists - @param target_source: string path, or - L{pkgcore.interfaces.data_source.base} derivative - @param data: mapping instance to write into the xpak. - @return: xpak instance - """ - try: - old_xpak = cls(target_source) - # force access - old_xpak.keys() - start = old_xpak.xpak_start - source_is_path = old_xpak._source_is_path - except (MalformedXpak, IOError): - source_is_path = isinstance(target_source, basestring) - if source_is_path: - try: - start = os.lstat(target_source).st_size - except OSError, e: - if e.errno != errno.ENOENT: - raise - start = 0 - else: - f = target_source.get_fileobj().seek(0, 2) - start = f.tell() - new_index = [] - new_data = [] - cur_pos = 0 - for key, val in data.iteritems(): - new_index.append(struct.pack(">L%isLL" % len(key), - len(key), key, cur_pos, len(val))) - new_data.append(val) - cur_pos += len(val) - - if source_is_path: - # rb+ required since A) binary, B) w truncates from the getgo - handle = open(target_source, "rb+") - else: - handle = target_source.get_fileobj() - - new_index = ''.join(new_index) - new_data = ''.join(new_data) - - handle.seek(start, 0) - # +12 is len(key) long, data_offset long, data_offset len long - handle.write(struct.pack(">%isLL%is%is%isL%is" % - (len(cls.header_pre_magic), - len(new_index), - len(new_data), - len(cls.trailer_pre_magic), - len(cls.trailer_post_magic)), - cls.header_pre_magic, - len(new_index), - len(new_data), - new_index, - new_data, - cls.trailer_pre_magic, - # the fun one; 16 for the footer, 8 for index/data longs, - # + index/data chunks. - len(new_index) + len(new_data) + 24, - cls.trailer_post_magic)) - - handle.truncate() - handle.close() - return Xpak(target_source) - - def _load_offsets(self): - fd = self._fd - index_start, index_len, data_len = self._check_magic(fd) - data_start = index_start + index_len - keys_dict = OrderedDict() - while index_len: - key_len = struct.unpack(">L", fd.read(4))[0] - key = fd.read(key_len) - if len(key) != key_len: - raise MalformedXpak( - "tried reading key %i of len %i, but hit EOF" % ( - len(keys_dict) + 1, key_len)) - try: - offset, data_len = struct.unpack(">LL", fd.read(8)) - except struct.error: - raise MalformedXpak( - "key %i, tried reading data offset/len but hit EOF" % ( - len(keys_dict) + 1)) - keys_dict[key] = (data_start + offset, data_len) - index_len -= (key_len + 12) # 12 for key_len, offset, data_len longs - - self._keys_dict = keys_dict - - def _check_magic(self, fd): - fd.seek(-16, 2) - try: - pre, size, post = struct.unpack( - self.trailer_parser, fd.read(self.trailer_size)) - if pre != self.trailer_pre_magic or post != self.trailer_post_magic: - raise MalformedXpak( - "not an xpak segment, trailer didn't match: %r" % fd) - except struct.error: - raise MalformedXpak( - "not an xpak segment, failed parsing trailer: %r" % fd) - - # this is a bit daft, but the format seems to intentionally - # have an off by 8 in the offset address. presumably cause the - # header was added after the fact, either way we go +8 to - # check the header magic. - fd.seek(-(size + 8), 2) - self.xpak_start = fd.tell() - try: - pre, index_len, data_len = struct.unpack( - self.header_parser, fd.read(self.header_size)) - if pre != self.header_pre_magic: - raise MalformedXpak( - "not an xpak segment, header didn't match: %r" % fd) - except struct.error: - raise MalformedXpak( - "not an xpak segment, failed parsing header: %r" % fd) - - return self.xpak_start + self.header_size, index_len, data_len - - def keys(self): - return list(self.iterkeys()) - - def values(self): - return list(self.itervalues()) - - def items(self): - return list(self.iteritems()) - - def __len__(self): - return len(self._keys_dict) - - def __contains__(self, key): - return key in self._keys_dict - - def __nonzero__(self): - return bool(self._keys_dict) - - def __iter__(self): - return iter(self._keys_dict) - - def iterkeys(self): - return self._keys_dict.iterkeys() - - def itervalues(self): - fd = self._fd - return (self._get_data(fd, *v) for v in self._keys_dict.itervalues()) - - def iteritems(self): - # note that it's an OrderedDict, so this works. - fd = self._fd - return ( - (k, self._get_data(fd, *v)) - for k, v in self._keys_dict.iteritems()) - - def __getitem__(self, key): - return self._get_data(self._fd, *self._keys_dict[key]) - - def __delitem__(self, key): - del self._keys_dict[key] - - def __setitem__(self, key, val): - self._keys_dict[key] = val - return val - - def get(self, key, default=None): - try: - return self[key] - except KeyError: - return default - - def pop(self, key, *a): - # faster then the exception form... - l = len(a) - if l > 1: - raise TypeError("pop accepts 1 or 2 args only") - if key in self._keys_dict: - o = self._keys_dict.pop(key) - elif l: - o = a[0] - else: - raise KeyError(key) - return o - - def _get_data(self, fd, offset, data_len): - # optimization for file objs; they cache tell position, but - # pass through all seek calls (nice, eh?) so we rely on that - # for cutting down on uneeded seeks; userland comparison being - # far cheaper then an actual syscall seek - if fd.tell() != offset: - fd.seek(offset, 0) - assert fd.tell() == offset - r = fd.read(data_len) - assert len(r) == data_len - return r diff --git a/pkgcore/cache/__init__.py b/pkgcore/cache/__init__.py deleted file mode 100644 index 2867bc2..0000000 --- a/pkgcore/cache/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright: 2005 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -""" -cache subsystem, typically used for storing package metadata -""" diff --git a/pkgcore/cache/anydbm.py b/pkgcore/cache/anydbm.py deleted file mode 100644 index 1f28c0b..0000000 --- a/pkgcore/cache/anydbm.py +++ /dev/null @@ -1,79 +0,0 @@ -# Copyright: 2005 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -""" -anydbm backend -""" - -anydbm_module = __import__("anydbm") -try: - import cPickle as pickle -except ImportError: - import pickle -import os -from pkgcore.cache import fs_template, errors - - -class database(fs_template.FsBased): - - """anydbm based cache backend, autocommiting""" - - autocommits = True - cleanse_keys = True - - def __init__(self, *args, **config): - self._db = None - super(database, self).__init__(*args, **config) - - default_db = config.get("dbtype","anydbm") - if not default_db.startswith("."): - default_db = '.' + default_db - - self._db_path = os.path.join( - self.location, fs_template.gen_label(self.label) + default_db) - self._db = None - - try: - self._db = anydbm_module.open(self._db_path, "w", self._perms) - except anydbm_module.error: - # XXX handle this at some point - try: - self._ensure_dirs() - self._ensure_dirs(self._db_path) - self._ensure_access(self._db_path) - except (OSError, IOError), e: - raise errors.InitializationError(self.__class__, e) - - # try again if failed - try: - if self._db is None: - self._db = anydbm_module.open( - self._db_path, "c", self._perms) - except anydbm_module.error, e: - raise errors.InitializationError(self.__class__, e) - __init__.__doc__ = fs_template.FsBased.__init__.__doc__ - - def iteritems(self): - return self._db.iteritems() - - def _getitem(self, cpv): - # we override getitem because it's just a cpickling of the - # data handed in. - return pickle.loads(self._db[cpv]) - - def _setitem(self, cpv, values): - self._db[cpv] = pickle.dumps(values, pickle.HIGHEST_PROTOCOL) - - def _delitem(self, cpv): - del self._db[cpv] - - def iterkeys(self): - return iter(self._db) - - def __contains__(self, cpv): - return cpv in self._db - - def __del__(self): - if self._db is not None: - self._db.sync() - self._db.close() diff --git a/pkgcore/cache/cdb.py b/pkgcore/cache/cdb.py deleted file mode 100644 index 0ad291a..0000000 --- a/pkgcore/cache/cdb.py +++ /dev/null @@ -1,109 +0,0 @@ -# Copyright: 2005 Gentoo Foundation -# Author(s): Jason Stubbs <jstubbs@gentoo.org> -# License: GPL2 - -""" -cdb backend -""" - -cdb_module = __import__("cdb") -try: - import cPickle as pickle -except ImportError: - import pickle - -import copy -import os - -from pkgcore.cache import fs_template, errors - - -class database(fs_template.FsBased): - - """cdb cache backend, non autocommiting""" - - autocommits = False - cleanse_keys = True - serialize_eclasses = False - - def __init__(self, *args, **config): - super(database, self).__init__(*args, **config) - - self._db_path = os.path.join( - self.location, fs_template.gen_label(self.label) + ".cdb") - self._db = None - try: - self._db = cdb_module.init(self._db_path) - - except cdb_module.error: - try: - self._ensure_dirs() - self._ensure_dirs(self._db_path) - self._ensure_access(self._db_path) - except (OSError, IOError), e: - raise errors.InitializationError(self.__class__, e) - - try: - cm = cdb_module.cdbmake(self._db_path, self._db_path+".tmp") - cm.finish() - self._ensure_access(self._db_path) - self._db = cdb_module.init(self._db_path) - except cdb_module.error, e: - raise errors.InitializationError(self.__class__, e) - self._adds = {} - self._dels = set() - __init__.__doc__ = fs_template.FsBased.__init__.__doc__ - - def iteritems(self): - self.commit() - return iter(self._db.each, None) - - def _getitem(self, cpv): - if cpv in self._adds: - d = copy.deepcopy(self._adds[cpv]) - else: - d = pickle.loads(self._db[cpv]) - return d - - def _setitem(self, cpv, values): - if cpv in self._dels: - del self._dels[cpv] - self._adds[cpv] = values - - def _delitem(self, cpv): - if cpv in self._adds: - del self._adds[cpv] - self._dels.add(cpv) - - def commit(self): - """commit any outstanding transactions""" - if not self._adds and not self._dels: - return - cm = cdb_module.cdbmake(self._db_path, self._db_path+str(os.getpid())) - for (key, value) in iter(self._db.each, None): - if key in self._dels: - del self._dels[key] - continue - if key in self._adds: - cm.add(key, pickle.dumps(self._adds.pop(key), - pickle.HIGHEST_PROTOCOL)) - else: - cm.add(key, value) - for (key, value) in self._adds.iteritems(): - cm.add(key, pickle.dumps(value, pickle.HIGHEST_PROTOCOL)) - cm.finish() - self._ensure_access(self._db_path) - self._db = cdb_module.init(self._db_path) - self._adds = {} - self._dels = {} - - def iterkeys(self): - self.commit() - return iter(self._db.keys()) - - def __contains__(self, cpv): - return cpv not in self._dels and (cpv in self._adds or cpv in self._db) - - def __del__(self): - if getattr(self, "_db", None): - self.commit() diff --git a/pkgcore/cache/errors.py b/pkgcore/cache/errors.py deleted file mode 100644 index f54c462..0000000 --- a/pkgcore/cache/errors.py +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright: 2005 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -""" -cache subsystem exceptions -""" - -class CacheError(Exception): - pass - -class InitializationError(CacheError): - def __init__(self, class_name, error): - CacheError.__init__(self, "Creation of instance %s failed due to %s" % - (class_name, error)) - self.error, self.class_name = error, class_name - - -class CacheCorruption(CacheError): - def __init__(self, key, ex): - CacheError.__init__(self, "%s is corrupt: %s" % (key, ex)) - self.key, self.ex = key, ex - - -class GeneralCacheCorruption(CacheError): - def __init__(self, ex): - CacheError.__init__(self, "corruption detected: %s" % (ex,)) - self.ex = ex - - -class InvalidRestriction(CacheError): - def __init__(self, key, restriction, exception=None): - if exception is None: - exception = '' - CacheError.__init__(self, "%s:%s is not valid: %s" % - (key, restriction, exception)) - self.key, self.restriction, self.ex = key, restriction, exception - - -class ReadOnly(CacheError): - def __init__(self, info=''): - CacheError.__init__(self, "cache is non-modifiable %s" % (info,)) - self.info = info diff --git a/pkgcore/cache/flat_hash.py b/pkgcore/cache/flat_hash.py deleted file mode 100644 index 0788243..0000000 --- a/pkgcore/cache/flat_hash.py +++ /dev/null @@ -1,136 +0,0 @@ -# Copyright: 2005-2007 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -""" -per key file based backend -""" - -import os, stat, errno -from pkgcore.cache import fs_template -from pkgcore.cache import errors -from pkgcore.config import ConfigHint -from snakeoil.osutils import join as pjoin, readlines - -class database(fs_template.FsBased): - - """ - stores cache entries in key=value form, stripping newlines - """ - - # TODO different way of passing in default auxdbkeys and location - pkgcore_config_type = ConfigHint( - {'readonly': 'bool', 'location': 'str', 'label': 'str', - 'auxdbkeys': 'list'}, - required=['location', 'label'], - positional=['location', 'label'], - typename='cache') - - - autocommits = True - - def __init__(self, *args, **config): - super(database, self).__init__(*args, **config) - self.location = self._format_location() - - if not os.path.exists(self.location): - self._ensure_dirs() - __init__.__doc__ = fs_template.FsBased.__init__.__doc__ - - def _format_location(self): - return pjoin(self.location, - self.label.lstrip(os.path.sep).rstrip(os.path.sep)) - - def _getitem(self, cpv): - path = pjoin(self.location, cpv) - try: - data = readlines(path, True, True, True) - except (IOError, OSError), e: - raise errors.CacheCorruption(cpv, e) - if data is None: - raise KeyError(cpv) - try: - d = self._parse_data(data, data.mtime) - except (OSError, ValueError), e: - raise errors.CacheCorruption(cpv, e) - return d - - def _parse_data(self, data, mtime): - d = self._cdict_kls() - known = self._known_keys - for x in data: - k, v = x.split("=", 1) - if k in known: - d[k] = v - - if self._mtime_used: - d["_mtime_"] = long(mtime) - return d - - def _setitem(self, cpv, values): - # might seem weird, but we rely on the trailing +1; this - # makes it behave properly for any cache depth (including no depth) - s = cpv.rfind("/") + 1 - fp = pjoin(self.location, - cpv[:s], ".update.%i.%s" % (os.getpid(), cpv[s:])) - try: - myf = open(fp, "w", 32768) - except IOError, ie: - if ie.errno == errno.ENOENT: - if not self._ensure_dirs(cpv): - raise errors.CacheCorruption( - cpv, 'error creating directory for %r' % (fp,)) - try: - myf = open(fp, "w", 32768) - except (OSError, IOError), e: - raise errors.CacheCorruption(cpv, e) - else: - raise errors.CacheCorruption(cpv, ie) - except OSError, e: - raise errors.CacheCorruption(cpv, e) - - for k, v in values.iteritems(): - if k != "_mtime_": - myf.writelines("%s=%s\n" % (k, v)) - - myf.close() - if self._mtime_used: - self._ensure_access(fp, mtime=values["_mtime_"]) - else: - self._ensure_access(fp) - - #update written. now we move it. - - new_fp = pjoin(self.location, cpv) - try: - os.rename(fp, new_fp) - except (OSError, IOError), e: - os.remove(fp) - raise errors.CacheCorruption(cpv, e) - - def _delitem(self, cpv): - try: - os.remove(pjoin(self.location, cpv)) - except OSError, e: - if e.errno == errno.ENOENT: - raise KeyError(cpv) - else: - raise errors.CacheCorruption(cpv, e) - - def __contains__(self, cpv): - return os.path.exists(pjoin(self.location, cpv)) - - def iterkeys(self): - """generator for walking the dir struct""" - dirs = [self.location] - len_base = len(self.location) - while dirs: - d = dirs.pop(0) - for l in os.listdir(d): - if l.endswith(".cpickle"): - continue - p = pjoin(d, l) - st = os.lstat(p) - if stat.S_ISDIR(st.st_mode): - dirs.append(p) - continue - yield p[len_base+1:] diff --git a/pkgcore/cache/fs_template.py b/pkgcore/cache/fs_template.py deleted file mode 100644 index 209fe59..0000000 --- a/pkgcore/cache/fs_template.py +++ /dev/null @@ -1,80 +0,0 @@ -# Copyright: 2005-2006 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -""" -template for fs based backends -""" - -import os -from pkgcore.cache import template -from pkgcore.os_data import portage_gid -from snakeoil.osutils import ensure_dirs - -class FsBased(template.database): - """Template wrapping fs needed options. - - Provides _ensure_access as a way to attempt to ensure files have - the specified owners/perms. - """ - - def __init__(self, *args, **config): - """ - throws InitializationError if needs args aren't specified - - @keyword gid: defaults to L{pkgcore.os_data.portage_gid}, - gid to force all entries to - @keyword perms: defaults to 0665, mode to force all entries to""" - - for x, y in (("gid", portage_gid), ("perms", 0664)): - if x in config: - setattr(self, "_"+x, config[x]) - del config[x] - else: - setattr(self, "_"+x, y) - super(FsBased, self).__init__(*args, **config) - - if self.label.startswith(os.path.sep): - # normpath. - self.label = os.path.sep + os.path.normpath( - self.label).lstrip(os.path.sep) - - self._mtime_used = "_mtime_" in self._known_keys - - __init__.__doc__ = "\n".join( - x.lstrip() for x in __init__.__doc__.split("\n") + [ - y.lstrip().replace("@param", "@keyword") - for y in template.database.__init__.__doc__.split("\n") - if "@param" in y]) - - def _ensure_access(self, path, mtime=-1): - """Ensure access to a path. - - @param mtime: if specified change mtime to this value. - @return: C{False} if unable to guarantee access, C{True} otherwise. - """ - try: - os.chown(path, -1, self._gid) - os.chmod(path, self._perms) - if mtime: - mtime = long(mtime) - os.utime(path, (mtime, mtime)) - except (OSError, IOError): - return False - return True - - def _ensure_dirs(self, path=None): - """Make sure a path relative to C{self.location} exists.""" - if path is not None: - path = self.location + os.path.sep + os.path.dirname(path) - else: - path = self.location - return ensure_dirs(path, mode=0775, minimal=False) - -def gen_label(label): - """Turn a user-defined label into something usable as a filename.""" - if label.find(os.path.sep) == -1: - return label - label = label.strip("\"").strip("'") - label = os.path.join(*(label.rstrip(os.path.sep).split(os.path.sep))) - tail = os.path.split(label)[1] - return "%s-%X" % (tail, abs(label.__hash__())) diff --git a/pkgcore/cache/metadata.py b/pkgcore/cache/metadata.py deleted file mode 100644 index 22db70c..0000000 --- a/pkgcore/cache/metadata.py +++ /dev/null @@ -1,201 +0,0 @@ -# Copyright: 2005-2006 Brian Harring <ferringb@gmail.com> -# Copyright: 2006 Marien Zwart <marienz@gentoo.org> -# License: GPL2 - -""" -cache backend designed for rsynced tree's pregenerated metadata. -""" - -import os -import errno -from pkgcore.cache import flat_hash, errors -from pkgcore.config import ConfigHint -from pkgcore.ebuild import eclass_cache -from snakeoil.osutils import join as pjoin -from snakeoil.mappings import ProtectedDict - - -# store the current key order *here*. -class database(flat_hash.database): - """ - Compatibility with (older) portage-generated caches. - - Autodetects per entry if it is a - L{flat_list<pkgcore.cache.flat_hash.database>} and flat_list - backends entry, and converts old (and incomplete) INHERITED field - to _eclasses_ as required. - """ - complete_eclass_entries = False - - auxdbkeys_order = ('DEPEND', 'RDEPEND', 'SLOT', 'SRC_URI', - 'RESTRICT', 'HOMEPAGE', 'LICENSE', 'DESCRIPTION', - 'KEYWORDS', 'INHERITED', 'IUSE', 'CDEPEND', - 'PDEPEND', 'PROVIDE', 'EAPI') - - # this is the old cache format, flat_list. hardcoded, and must - # remain that way. - magic_line_count = 22 - - autocommits = True - - def __init__(self, location, *args, **config): - self.base_loc = location - super(database, self).__init__(location, *args, **config) - self.ec = eclass_cache.cache(pjoin(self.base_loc, "eclass"), - self.base_loc) - self.hardcoded_auxdbkeys_order = tuple((idx, key) - for idx, key in enumerate(self.auxdbkeys_order) - if key in self._known_keys) - - __init__.__doc__ = flat_hash.database.__init__.__doc__.replace( - "@keyword location", "@param location") - - - def _format_location(self): - return pjoin(self.location, "metadata", "cache") - - def __getitem__(self, cpv): - d = flat_hash.database.__getitem__(self, cpv) - - if "_eclasses_" not in d: - if "INHERITED" in d: - d["_eclasses_"] = self.ec.get_eclass_data( - d["INHERITED"].split()) - del d["INHERITED"] - else: - d["_eclasses_"] = self.reconstruct_eclasses(cpv, d["_eclasses_"]) - - return d - - def _parse_data(self, data, mtime): - data = list(data) - if len(data) != self.magic_line_count: - raise errors.GeneralCacheCorruption( - "wrong line count, requires %i, got %i" % - (self.magic_line_count, len(data))) - - # this one's interesting. - d = self._cdict_kls() - for idx, key in self.hardcoded_auxdbkeys_order: - d[key] = data[idx].strip() - - if self._mtime_used: - d["_mtime_"] = mtime - return d - - def _setitem(self, cpv, values): - values = ProtectedDict(values) - - # hack. proper solution is to make this a __setitem__ override, since - # template.__setitem__ serializes _eclasses_, then we reconstruct it. - eclasses = values.pop('_eclasses_', None) - if eclasses is not None: - eclasses = self.reconstruct_eclasses(cpv, eclasses) - values["INHERITED"] = ' '.join(eclasses) - - s = cpv.rfind("/") - fp = pjoin( - self.location, cpv[:s],".update.%i.%s" % (os.getpid(), cpv[s+1:])) - try: - myf = open(fp, "w") - except (OSError, IOError), e: - if errno.ENOENT == e.errno: - try: - self._ensure_dirs(cpv) - myf = open(fp,"w") - except (OSError, IOError),e: - raise errors.CacheCorruption(cpv, e) - else: - raise errors.CacheCorruption(cpv, e) - - count = 0 - for idx, key in self.hardcoded_auxdbkeys_order: - myf.write("%s%s" % ("\n" * (idx - count), values.get(key, ""))) - count = idx - myf.write("\n" * (self.magic_line_count - count)) - - myf.close() - self._set_mtime(fp, values, eclasses) - - #update written. now we move it. - new_fp = pjoin(self.location, cpv) - try: - os.rename(fp, new_fp) - except (OSError, IOError), e: - os.remove(fp) - raise errors.CacheCorruption(cpv, e) - - def _set_mtime(self, fp, values, eclasses): - if self._mtime_used: - self._ensure_access(fp, mtime=values["_mtime_"]) - - -class paludis_flat_list(database): - - """ - (Hopefully) write a paludis specific form of flat_list format cache. - Not very well tested. - - difference from a normal flat_list cache is that mtime is set to ebuild - for normal, for paludis it's max mtime of eclasses/ebuild involved. - """ - - pkgcore_config_type = ConfigHint( - {'readonly': 'bool', 'location': 'str', 'label': 'str'}, - required=['location', 'label'], - positional=['location', 'label'], - typename='cache') - - def __init__(self, location, *args, **config): - config['auxdbkeys'] = self.auxdbkeys_order - database.__init__(self, location, *args, **config) - - def _set_mtime(self, fp, values, eclasses): - mtime = values.get("_mtime_", 0) - - if eclasses: - self._ensure_access( - fp, - mtime=max(max(mtime for path, mtime in eclasses.itervalues()), - mtime)) - else: - self._ensure_access(fp, mtime) - - -class protective_database(database): - - def _parse_data(self, data, mtime): - # easy attempt first. - data = list(data) - if len(data) != self.magic_line_count: - return flat_hash.database._parse_data(self, data, mtime) - - # this one's interesting. - d = self._cdict_kls() - - for line in data: - # yes, meant to iterate over a string. - hashed = False - for idx, c in enumerate(line): - if not c.isalpha(): - if c == "=" and idx > 0: - hashed = True - d[line[:idx]] = line[idx + 1:] - elif c == "_" or c.isdigit(): - continue - break - elif not c.isupper(): - break - - if not hashed: - # non hashed. - d.clear() - for idx, key in self.hardcoded_auxdbkeys_order: - d[key] = data[idx].strip() - break - - if self._mtime_used: - d["_mtime_"] = mtime - return d - - diff --git a/pkgcore/cache/sql_template.py b/pkgcore/cache/sql_template.py deleted file mode 100644 index ad8a709..0000000 --- a/pkgcore/cache/sql_template.py +++ /dev/null @@ -1,327 +0,0 @@ -# Copyright: 2005 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -""" -template for sql backends- needs work -""" - -from pkgcore.cache import template, errors - -class SQLDatabase(template.database): - """template class for RDBM based caches - - This class is designed such that derivatives don't have to change - much code, mostly constant strings. - - _BaseError must be an exception class that all Exceptions thrown - from the derived RDBMS are derived from. - - SCHEMA_INSERT_CPV_INTO_PACKAGE should be modified dependant on the - RDBMS, as should SCHEMA_PACKAGE_CREATE- basically you need to deal - with creation of a unique pkgid. If the dbapi2 rdbms class has a - method of recovering that id, then modify _insert_cpv to remove - the extra select. - - Creation of a derived class involves supplying _initdb_con, and - table_exists. Additionally, the default schemas may have to be - modified. - """ - - SCHEMA_PACKAGE_NAME = "package_cache" - SCHEMA_PACKAGE_CREATE = ( - "CREATE TABLE %s ( " - "pkgid INTEGER PRIMARY KEY, label VARCHAR(255), cpv VARCHAR(255), " - "UNIQUE(label, cpv))" % SCHEMA_PACKAGE_NAME) - SCHEMA_PACKAGE_DROP = "DROP TABLE %s" % SCHEMA_PACKAGE_NAME - - SCHEMA_VALUES_NAME = "values_cache" - SCHEMA_VALUES_CREATE = ( - "CREATE TABLE %s ( " - "pkgid integer references %s (pkgid) on delete cascade, " - "key varchar(255), value text, UNIQUE(pkgid, key))" % ( - SCHEMA_VALUES_NAME, SCHEMA_PACKAGE_NAME)) - SCHEMA_VALUES_DROP = "DROP TABLE %s" % SCHEMA_VALUES_NAME - SCHEMA_INSERT_CPV_INTO_PACKAGE = ( - "INSERT INTO %s (label, cpv) VALUES(%%s, %%s)" % SCHEMA_PACKAGE_NAME) - - _BaseError = () - _dbClass = None - - autocommits = False -# cleanse_keys = True - - # boolean indicating if the derived RDBMS class supports replace syntax - _supports_replace = False - - def __init__(self, location, label, auxdbkeys, *args, **config): - """initialize the instance. - derived classes shouldn't need to override this""" - - self.db = None - super(SQLDatabase, self).__init__(location, label, auxdbkeys, - *args, **config) - - config.setdefault("host","127.0.0.1") - config.setdefault("autocommit", self.autocommits) - self._initdb_con(config) - - self.label = self._sfilter(self.label) - - def _dbconnect(self, config): - """Initialize the internal db connection and cursor. - - Should be overridden if the derived class needs special - parameters for initializing the db connection or cursor. - """ - self.db = self._dbClass(**config) - self.con = self.db.cursor() - - def _initdb_con(self, config): - """ensure needed tables are in place. - - If the derived class needs a different set of table creation - commands, overload the approriate SCHEMA_ attributes. If it - needs additional execution beyond that, override this. - """ - - self._dbconnect(config) - if not self._table_exists(self.SCHEMA_PACKAGE_NAME): - if self.readonly: - raise errors.ReadOnly( - "table %s doesn't exist" % self.SCHEMA_PACKAGE_NAME) - try: - self.con.execute(self.SCHEMA_PACKAGE_CREATE) - except self._BaseError, e: - raise errors.InitializationError(self.__class__, e) - - if not self._table_exists(self.SCHEMA_VALUES_NAME): - if self.readonly: - raise errors.ReadOnly( - "table %s doesn't exist" % self.SCHEMA_VALUES_NAME) - try: - self.con.execute(self.SCHEMA_VALUES_CREATE) - except self._BaseError, e: - raise errors.InitializationError(self.__class__, e) - - def _table_exists(self, tbl): - """return true if a table exists - derived classes must override this""" - raise NotImplementedError - - def _sfilter(self, string): - """meta escaping, returns quoted string for use in sql statements""" - return "\"%s\"" % string.replace("\\","\\\\").replace("\"","\\\"") - - def _getitem(self, cpv): - try: - self.con.execute( - "SELECT key, value FROM %s NATURAL JOIN %s " - "WHERE label=%s AND cpv=%s" % ( - self.SCHEMA_PACKAGE_NAME, self.SCHEMA_VALUES_NAME, - self.label, self._sfilter(cpv))) - except self._BaseError, e: - raise errors.CacheCorruption(self, cpv, e) - - rows = self.con.fetchall() - - if not rows: - raise KeyError(cpv) - - vals = dict((k,"") for k in self._known_keys) - vals.update(dict(rows)) - return vals - - def _delitem(self, cpv): - """delete a cpv cache entry - derived RDBM classes for this *must* either support cascaded deletes, or - override this method - """ - - try: - try: - self.con.execute("DELETE FROM %s WHERE label=%s AND cpv=%s" % \ - (self.SCHEMA_PACKAGE_NAME, self.label, self._sfilter(cpv))) - if self.autocommits: - self.commit() - except self._BaseError, e: - raise errors.CacheCorruption(self, cpv, e) - if self.con.rowcount <= 0: - raise KeyError(cpv) - except Exception: - if not self.autocommits: - self.db.rollback() - # yes, this can roll back a lot more then just the delete. deal. - raise - - def __del__(self): - if self.db is not None: - self.commit() - self.db.close() - - def _setitem(self, cpv, values): - try: - # insert. - try: - pkgid = self._insert_cpv(cpv) - except self._BaseError, e: - raise errors.CacheCorruption(cpv, e) - - # __getitem__ fills out missing values, - # so we store only what's handed to us and is a known key - db_values = [] - for key in self._known_keys: - if values.get(key, "") != "": - db_values.append({"key":key, "value":values[key]}) - - if db_values: - try: - self.con.executemany( - "INSERT INTO %s (pkgid, key, value) " - "VALUES(\"%s\", %%(key)s, %%(value)s)" % - (self.SCHEMA_VALUES_NAME, str(pkgid)), db_values) - except self._BaseError, e: - raise errors.CacheCorruption(cpv, e) - if self.autocommits: - self.commit() - - except Exception: - if not self.autocommits: - try: - self.db.rollback() - except self._BaseError: - pass - raise - - - def _insert_cpv(self, cpv): - """Insert a cpv in the db. - - Uses SCHEMA_INSERT_CPV_INTO_PACKAGE, which must be overloaded - if the table definition doesn't support auto-increment columns - for pkgid. - - @return: the cpvs new pkgid - - note this doesn't commit the transaction. The caller is expected to. - """ - - cpv = self._sfilter(cpv) - if self._supports_replace: - query_str = self.SCHEMA_INSERT_CPV_INTO_PACKAGE.replace( - "INSERT", "REPLACE", 1) - else: - # just delete it. - try: - del self[cpv] - except (errors.CacheCorruption, KeyError): - pass - query_str = self.SCHEMA_INSERT_CPV_INTO_PACKAGE - - try: - self.con.execute(query_str % (self.label, cpv)) - except self._BaseError: - self.db.rollback() - raise - - self.con.execute("SELECT pkgid FROM %s WHERE label=%s AND cpv=%s" % \ - (self.SCHEMA_PACKAGE_NAME, self.label, cpv)) - - if self.con.rowcount != 1: - raise errors.CacheCorruption( - cpv, "Tried to insert the cpv, but found " - " %i matches upon the following select!" % ( - self.con.rowcount)) - return self.con.fetchone()[0] - - def __contains__(self, cpv): - if not self.autocommits: - try: - self.commit() - except self._BaseError, e: - raise errors.GeneralCacheCorruption(e) - - try: - self.con.execute("SELECT cpv FROM %s WHERE label=%s AND cpv=%s" % \ - (self.SCHEMA_PACKAGE_NAME, self.label, self._sfilter(cpv))) - except self._BaseError, e: - raise errors.GeneralCacheCorruption(e) - return self.con.rowcount > 0 - - def iterkeys(self): - if not self.autocommits: - try: - self.commit() - except self._BaseError, e: - raise errors.GeneralCacheCorruption(e) - - try: - self.con.execute("SELECT cpv FROM %s WHERE label=%s" % - (self.SCHEMA_PACKAGE_NAME, self.label)) - except self._BaseError, e: - raise errors.GeneralCacheCorruption(e) - - for x in self.con.fetchall(): - yield x[0] - - def iteritems(self): - try: - self.con.execute( - "SELECT cpv, key, value FROM %s NATURAL JOIN %s WHERE label=%s" - % (self.SCHEMA_PACKAGE_NAME, self.SCHEMA_VALUES_NAME, - self.label)) - except self._BaseError, e: - # XXX this makes no sense - raise errors.CacheCorruption(self, 'iteritems', e) - - oldcpv = None - l = [] - for cpv, key, v in self.con.fetchall(): - if oldcpv != cpv: - if oldcpv is not None: - d = dict(l) - if "_eclasses_" in d: - d["_eclasses_"] = self.reconstruct_eclasses( - oldcpv, d["_eclasses_"]) - yield oldcpv, d - l = [] - oldcpv = cpv - l.append((key, v)) - - if oldcpv is not None: - d = dict(l) - if "_eclasses_" in d: - d["_eclasses_"] = self.reconstruct_eclasses( - oldcpv, d["_eclasses_"]) - yield oldcpv, d - - def commit(self): - self.db.commit() - - def get_matches(self, match_dict): - query_list = [] - for k, v in match_dict.items(): - if k not in self._known_keys: - raise errors.InvalidRestriction( - k, v, "key isn't known to this cache instance") - v = v.replace("%","\\%") - v = v.replace(".*","%") - query_list.append( - "(key=%s AND value LIKE %s)" % ( - self._sfilter(k), self._sfilter(v))) - - if query_list: - query = " AND "+" AND ".join(query_list) - else: - query = '' - - print ( - "query = SELECT cpv from package_cache natural join values_cache " - "WHERE label=%s %s" % (self.label, query)) - try: - self.con.execute( - "SELECT cpv from package_cache natural join values_cache " - "WHERE label=%s %s" % (self.label, query)) - except self._BaseError, e: - raise errors.GeneralCacheCorruption(e) - - return [ row[0] for row in self.con.fetchall() ] diff --git a/pkgcore/cache/sqlite.py b/pkgcore/cache/sqlite.py deleted file mode 100644 index dbcabbc..0000000 --- a/pkgcore/cache/sqlite.py +++ /dev/null @@ -1,80 +0,0 @@ -# Copyright: 2005 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -""" -pysqlite <v2 backend -""" - -import os -from pkgcore.cache import sql_template, fs_template, errors -sqlite_module = __import__("sqlite") - -class database(fs_template.FsBased, sql_template.SQLDatabase): - - SCHEMA_DELETE_NAME = "delete_package_values" - SCHEMA_DELETE_TRIGGER = """CREATE TRIGGER %s AFTER DELETE on %s - begin - DELETE FROM %s WHERE pkgid=old.pkgid; - end;""" % (SCHEMA_DELETE_NAME, sql_template.SQLDatabase.SCHEMA_PACKAGE_NAME, - sql_template.SQLDatabase.SCHEMA_VALUES_NAME) - - _BaseError = sqlite_module.Error - _dbClass = sqlite_module - _supports_replace = True - - def _dbconnect(self, config): - self._dbpath = os.path.join( - self.location, fs_template.gen_label(self.label) + ".sqldb") - try: - self.db = sqlite_module.connect( - self._dbpath, mode=self._perms, autocommit=False) - if not self._ensure_access(self._dbpath): - raise errors.InitializationError( - self.__class__, "can't ensure perms on %s" % self._dbpath) - self.con = self.db.cursor() - except self._BaseError, e: - raise errors.InitializationError(self.__class__, e) - - def _initdb_con(self, config): - sql_template.SQLDatabase._initdb_con(self, config) - try: - self.con.execute( - "SELECT name FROM sqlite_master " - "WHERE type=\"trigger\" AND name=%s" % - self._sfilter(self.SCHEMA_DELETE_NAME)) - if self.con.rowcount == 0: - self.con.execute(self.SCHEMA_DELETE_TRIGGER) - self.db.commit() - except self._BaseError, e: - raise errors.InitializationError(self.__class__, e) - - def _table_exists(self, tbl): - """return true/false dependant on a tbl existing""" - try: - self.con.execute( - "SELECT name FROM sqlite_master " - "WHERE type=\"table\" AND name=%s" % - self._sfilter(tbl)) - except self._BaseError: - # XXX crappy. - return False - return len(self.con.fetchall()) == 1 - - # we can do it minus a query via rowid. - def _insert_cpv(self, cpv): - cpv = self._sfilter(cpv) - try: - self.con.execute( - self.SCHEMA_INSERT_CPV_INTO_PACKAGE.replace( - "INSERT", "REPLACE", 1) % - (self.label, cpv)) - except self._BaseError, e: - raise errors.CacheCorruption( - cpv, "tried to insert a cpv, but failed: %s" % str(e)) - - # sums the delete also - if self.con.rowcount <= 0 or self.con.rowcount > 2: - raise errors.CacheCorruption( - cpv, "tried to insert a cpv, but failed- %i rows modified" % - self.rowcount) - return self.con.lastrowid diff --git a/pkgcore/cache/template.py b/pkgcore/cache/template.py deleted file mode 100644 index a1ac94e..0000000 --- a/pkgcore/cache/template.py +++ /dev/null @@ -1,236 +0,0 @@ -# Copyright: 2005-2006 Brian Harring <ferringb@gmail.com> -# Author(s): Brian Harring <ferringb@gmail.com> -# License: GPL2 - -""" -template for cache backend classes -""" - -from pkgcore.cache import errors -from snakeoil.mappings import ProtectedDict -from snakeoil.obj import make_SlottedDict_kls - -# temp hack for .2 -from pkgcore.ebuild.const import metadata_keys -metadata_keys = tuple(metadata_keys) - -class database(object): - # this is for metadata/cache transfer. - # basically flags the cache needs be updated when transfered cache to cache. - # leave this. - - """ - @ivar autocommits: Controls whether the template commits every update, - or queues up updates. - @ivar complete_eclass_entries: Specifies if the cache backend stores full - eclass data, or partial. - @ivar cleanse_keys: Boolean controlling whether the template should drop - empty keys for storing. - @ivar serialize_eclasses: Boolean controlling whether the template should - serialize eclass data itself, or leave it to the derivative. - """ - - complete_eclass_entries = True - autocommits = False - cleanse_keys = False - serialize_eclasses = True - - def __init__(self, location, label, auxdbkeys=metadata_keys, - readonly=False): - """ - initialize the derived class; specifically, store label/keys - - @param location: fs location the cache is stored at - @param label: cache label - @param auxdbkeys: sequence of allowed keys for each cache entry - @param readonly: defaults to False, - controls whether the cache is mutable. - """ - self._known_keys = frozenset(auxdbkeys) - self._cdict_kls = make_SlottedDict_kls(self._known_keys) - self.location = location - self.label = label - self.readonly = readonly - self.sync_rate = 0 - self.updates = 0 - - def __getitem__(self, cpv): - """set a cpv to values - - This shouldn't be overriden in derived classes since it - handles the __eclasses__ conversion. That said, if the class - handles it, they can override it. - """ - if self.updates > self.sync_rate: - self.commit() - self.updates = 0 - d = self._getitem(cpv) - if self.serialize_eclasses and "_eclasses_" in d: - d["_eclasses_"] = self.reconstruct_eclasses(cpv, d["_eclasses_"]) - return d - - def _getitem(self, cpv): - """get cpv's values. - - override this in derived classess. - """ - raise NotImplementedError - - def __setitem__(self, cpv, values): - """set a cpv to values - - This shouldn't be overriden in derived classes since it - handles the readonly checks. - """ - if self.readonly: - raise errors.ReadOnly() - if self.cleanse_keys: - d = ProtectedDict(values) - for k in d.iterkeys(): - if not d[k]: - del d[k] - if self.serialize_eclasses and "_eclasses_" in values: - d["_eclasses_"] = self.deconstruct_eclasses(d["_eclasses_"]) - elif self.serialize_eclasses and "_eclasses_" in values: - d = ProtectedDict(values) - d["_eclasses_"] = self.deconstruct_eclasses(d["_eclasses_"]) - else: - d = values - self._setitem(cpv, d) - if not self.autocommits: - self.updates += 1 - if self.updates > self.sync_rate: - self.commit() - self.updates = 0 - - def _setitem(self, name, values): - """__setitem__ calls this after readonly checks. - - override it in derived classes. - note _eclasses_ key *must* be handled. - """ - raise NotImplementedError - - def __delitem__(self, cpv): - """delete a key from the cache. - - This shouldn't be overriden in derived classes since it - handles the readonly checks. - """ - if self.readonly: - raise errors.ReadOnly() - if not self.autocommits: - self.updates += 1 - self._delitem(cpv) - if self.updates > self.sync_rate: - self.commit() - self.updates = 0 - - def _delitem(self, cpv): - """__delitem__ calls this after readonly checks. - - override it in derived classes. - """ - raise NotImplementedError - - def __contains__(self, cpv): - raise NotImplementedError - - def has_key(self, cpv): - return cpv in self - - def keys(self): - return list(self.iterkeys()) - - def iterkeys(self): - raise NotImplementedError - - def iteritems(self): - for x in self.iterkeys(): - yield (x, self[x]) - - def items(self): - return list(self.iteritems()) - - def sync(self, rate=0): - self.sync_rate = rate - if rate == 0: - self.commit() - - def commit(self): - if not self.autocommits: - raise NotImplementedError - - def get_matches(self, match_dict): - """ - Generic function for walking the entire cache db, matching - restrictions to filter what cpv's are returned. - - Derived classes should override this if they can implement a - faster method then pulling each cpv:values, and checking it. - For example, RDBMS derived classes should push the matching - logic down to the actual RDBM. - """ - - import re - restricts = {} - for key, match in match_dict.iteritems(): - # XXX this sucks. - try: - if isinstance(match, str): - restricts[key] = re.compile(match).match - else: - restricts[key] = re.compile(match[0], match[1]).match - except re.error, e: - raise errors.InvalidRestriction(key, match, e) - if key not in self._known_keys: - raise errors.InvalidRestriction(key, match, - "Key isn't valid") - - for cpv, vals in self.iteritems(): - for key, match in restricts.iteritems(): - if not match(vals[key]): - break - else: - yield cpv - - @staticmethod - def deconstruct_eclasses(eclass_dict): - """takes a dict, returns a string representing said dict""" - return "\t".join( - "%s\t%s\t%s" % (k, v[0], v[1]) - for k, v in eclass_dict.iteritems()) - - @staticmethod - def reconstruct_eclasses(cpv, eclass_string): - """Turn a string from L{serialize_eclasses} into a dict.""" - if not isinstance(eclass_string, basestring): - raise TypeError("eclass_string must be basestring, got %r" % - eclass_string) - eclasses = eclass_string.strip().split("\t") - if eclasses == [""]: - # occasionally this occurs in the fs backends. they suck. - return {} - - l = len(eclasses) - if not l % 3: - paths = True - elif not l % 2: - # edge case of a multiple of 6 - paths = not eclasses[1].isdigit() - else: - raise errors.CacheCorruption( - cpv, "_eclasses_ was of invalid len %i" - "(must be mod 3 or mod 2)" % len(eclasses)) - d = {} - try: - if paths: - for x in xrange(0, len(eclasses), 3): - d[eclasses[x]] = (eclasses[x + 1], long(eclasses[x + 2])) - else: - for x in xrange(0, len(eclasses), 2): - d[eclasses[x]] = ('', long(eclasses[x + 1])) - except ValueError: - raise errors.CacheCorruption( - cpv, 'ValueError reading %r' % (eclass_string,)) - return d diff --git a/pkgcore/cache/util.py b/pkgcore/cache/util.py deleted file mode 100644 index fb00adb..0000000 --- a/pkgcore/cache/util.py +++ /dev/null @@ -1,118 +0,0 @@ -# Copyright: 2005 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -""" -cache backend utilities -""" - -from pkgcore.cache import errors - -def mirror_cache(valid_nodes_iterable, src_cache, trg_cache, eclass_cache=None, - verbose_instance=None): - """ - make a cache backend a mirror of another - - @param valid_nodes_iterable: valid keys - @param src_cache: L{pkgcore.cache.template.database} instance - to copy keys from - @param trg_cache: L{pkgcore.cache.template.database} instance - to write keys to - @param eclass_cache: if doing eclass_cache translation, - a L{pkgcore.ebuild.eclass_cache.cache} instance to use, else None - @param verbose_instance: either None (defaulting to L{quiet_mirroring}), - or a L{quiet_mirroring} derivative - """ - - if not src_cache.complete_eclass_entries and not eclass_cache: - raise Exception( - "eclass_cache required for cache's of class %s!" % - src_cache.__class__) - - if verbose_instance is None: - noise = quiet_mirroring() - else: - noise = verbose_instance - - dead_nodes = set(trg_cache.iterkeys()) - count = 0 - - if not trg_cache.autocommits: - trg_cache.sync(100) - - for x in valid_nodes_iterable: - count += 1 - if x in dead_nodes: - dead_nodes.remove(x) - try: - entry = src_cache[x] - except KeyError: - noise.missing_entry(x) - continue - if entry.get("INHERITED",""): - if src_cache.complete_eclass_entries: - if not "_eclasses_" in entry: - noise.corruption(x,"missing _eclasses_ field") - continue - if not eclass_cache.is_eclass_data_valid(entry["_eclasses_"]): - noise.eclass_stale(x) - continue - else: - entry["_eclasses_"] = eclass_cache.get_eclass_data( - entry["INHERITED"].split(), from_master_only=True) - if not entry["_eclasses_"]: - noise.eclass_stale(x) - continue - - # by this time, if it reaches here, the eclass has been - # validated, and the entry has been updated/translated (if - # needs be, for metadata/cache mainly) - try: - trg_cache[x] = entry - except errors.CacheError, ce: - noise.exception(x, ce) - del ce - continue - - if count >= noise.call_update_min: - noise.update(x) - count = 0 - - if not trg_cache.autocommits: - trg_cache.commit() - - # ok. by this time, the trg_cache is up to date, and we have a - # dict with a crapload of cpv's. we now walk the target db, - # removing stuff if it's in the list. - for key in dead_nodes: - try: - del trg_cache[key] - except errors.CacheError, ce: - noise.exception(ce) - del ce - - -# "More than one statement on a single line" -# pylint: disable-msg=C0321 - -class quiet_mirroring(object): - """Quiet mirror_cache callback object for getting progress information.""" - # call_update_every is used by mirror_cache to determine how often - # to call in. quiet defaults to 2^24 -1. Don't call update, 'cept - # once every 16 million or so :) - call_update_min = 0xffffff - def update(self, key, *arg): pass - def exception(self, key, *arg): pass - def eclass_stale(self, *arg): pass - def missing_entry(self, key): pass - def misc(self, key, *arg): pass - def corruption(self, key, s): pass - -class non_quiet_mirroring(quiet_mirroring): - """prints to stdout each step in cache mirroring""" - - call_update_min = 1 - def update(self, key, *arg): print "processed", key - def exception(self, key, *arg): print "exec", key, arg - def missing(self, key): print "key %s is missing", key - def corruption(self, key, *arg): print "corrupt %s:" % key, arg - def eclass_stale(self, key, *arg): print "stale %s:" % key, arg diff --git a/pkgcore/chksum/__init__.py b/pkgcore/chksum/__init__.py deleted file mode 100644 index 3978273..0000000 --- a/pkgcore/chksum/__init__.py +++ /dev/null @@ -1,116 +0,0 @@ -# Copyright: 2005 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -""" -chksum verification/generation subsystem -""" - -from pkgcore.interfaces.data_source import base as base_data_source -from snakeoil.demandload import demandload -demandload(globals(), "os", - "sys", - "pkgcore.log:logger", - "pkgcore.chksum.defaults:loop_over_file", - "snakeoil.modules:load_module", - "snakeoil.osutils:listdir_files", -) - -chksum_types = {} -__inited__ = False - -def get_handler(requested): - - """ - get a chksum handler - - @raise KeyError: if chksum type has no registered handler - @return: chksum handler (callable) - """ - - if not __inited__: - init() - if requested not in chksum_types: - raise KeyError("no handler for %s" % requested) - return chksum_types[requested] - - -def get_handlers(requested=None): - - """ - get chksum handlers - - @param requested: None (all handlers), or a sequence of the specific - handlers desired. - @raise KeyError: if requested chksum type has no registered handler - @return: dict of chksum_type:chksum handler - """ - - if requested is None: - if not __inited__: - init() - return dict(chksum_types) - d = {} - for x in requested: - d[x] = get_handler(x) - return d - - -def init(additional_handlers=None): - - """ - init the chksum subsystem. - - Scan the dir, find what handlers are available, etc. - - @param additional_handlers: None, or pass in a dict of type:func - """ - - global __inited__ - - if additional_handlers is not None and not isinstance( - additional_handlers, dict): - raise TypeError("additional handlers must be a dict!") - - chksum_types.clear() - __inited__ = False - loc = os.path.dirname(sys.modules[__name__].__file__) - for f in listdir_files(loc): - if not f.endswith(".py") or f.startswith("__init__."): - continue - try: - i = f.find(".") - if i != -1: - f = f[:i] - del i - m = load_module(__name__+"."+f) - except ImportError: - continue - try: - types = getattr(m, "chksum_types") - except AttributeError: - # no go. - continue - try: - chksum_types.update(types) - - except ValueError: - logger.warn( - "%s.%s invalid chksum_types, ValueError Exception" % ( - __name__, f)) - continue - - if additional_handlers is not None: - chksum_types.update(additional_handlers) - - __inited__ = True - - -def get_chksums(location, *chksums): - """ - run multiple chksumers over a data_source/file path - """ - handlers = get_handlers(chksums) - # try to hand off to the per file handler, may be faster. - if len(chksums) == 1: - return [handlers[chksums[0]](location)] - return loop_over_file(location, *[handlers[k].new() for k in chksums]) diff --git a/pkgcore/chksum/defaults.py b/pkgcore/chksum/defaults.py deleted file mode 100644 index 57f0540..0000000 --- a/pkgcore/chksum/defaults.py +++ /dev/null @@ -1,288 +0,0 @@ -# Copyright: 2006-2007 Brian Harring <ferringb@gmail.com> -# License: GPL2 - - -""" -default chksum handlers implementation- sha1, sha256, rmd160, and md5 -""" -from pkgcore.interfaces.data_source import base as base_data_source -from snakeoil.currying import partial -from snakeoil import modules -from snakeoil.demandload import demandload -demandload(globals(), "os") - -blocksize = 32768 - -sha1_size = 40 -md5_size = 32 -rmd160_size = 40 -sha256_size = 64 - -def loop_over_file(filename, *objs): - if isinstance(filename, base_data_source): - if filename.get_path is not None: - filename = filename.get_path() - else: - filename = filename.get_fileobj() - wipeit = False - if isinstance(filename, basestring): - wipeit = True - f = open(filename, 'rb', blocksize * 2) - else: - f = filename - # reposition to start - f.seek(0, 0) - try: - data = f.read(blocksize) - # XXX why is size tracked here? It seems to be unused... - size = 0L - chfs = [chf() for chf in objs] - while data: - for chf in chfs: - chf.update(data) - size = size + len(data) - data = f.read(blocksize) - - return [long(chf.hexdigest(), 16) for chf in chfs] - finally: - if wipeit: - f.close() - - -class Chksummer(object): - - def __init__(self, chf_type, obj, str_size): - self.obj = obj - self.chf_type = chf_type - self.str_size = str_size - - def new(self): - return self.obj - - def long2str(self, val): - return ("%x" % val).rjust(self.str_size, '0') - - @staticmethod - def str2long(val): - return long(val, 16) - - def __call__(self, filename): - return loop_over_file(filename, self.obj)[0] - - def __str__(self): - return "%s chksummer" % self.chf_type - - -# We have a couple of options: -# -# - If we are on python 2.5 or newer we can use hashlib, which uses -# openssl if available (this will be fast and support a whole bunch -# of hashes) and use a c implementation from python itself otherwise -# (does not support as many hashes, slower). -# - On older pythons we can use the sha and md5 module for sha1 and md5. -# On python 2.5 these are deprecated wrappers around hashlib. -# - On any python we can use the fchksum module (if available) which can -# hash an entire file faster than we can, probably just because it does the -# file-reading and hashing both in c. -# - For any python we can use PyCrypto. Supports many hashes, fast but not -# as fast as openssl-powered hashlib. Not compared to cpython hashlib. -# -# To complicate matters hashlib has a couple of hashes always present -# as attributes of the hashlib module and less common hashes available -# through a constructor taking a string. The former is faster. -# -# Some timing data from my athlonxp 2600+, python 2.4.3, python 2.5rc1, -# pycrypto 2.0.1-r5, openssl 0.9.7j, fchksum 1.7.1 (not exhaustive obviously): -# (test file is the Python 2.4.3 tarball, 7.7M) -# -# python2.4 -m timeit -s 'import fchksum' -# 'fchksum.fmd5t("/home/marienz/tmp/Python-2.4.3.tar.bz2")[0]' -# 40 +/- 1 msec roughly -# -# same with python2.5: same results. -# -# python2.4 -m timeit -s 'from pkgcore.chksum import defaults;import md5' -# 'defaults.loop_over_file(md5, "/home/marienz/tmp/Python-2.4.3.tar.bz2")' -# 64 +/- 1 msec roughly -# -# Same with python2.5: -# 37 +/- 1 msec roughly -# -# python2.5 -m timeit -s -# 'from pkgcore.chksum import defaults; from snakeoil import currying;' -# -s 'import hashlib; hash = currying.pre_curry(hashlib.new, "md5")' -# 'defaults.loop_over_file(hash, "/home/marienz/tmp/Python-2.4.3.tar.bz2")' -# 37 +/- 1 msec roughly -# -# python2.5 -m timeit -s 'import hashlib' -# 'h=hashlib.new("md5"); h.update("spork"); h.hexdigest()' -# 6-7 usec per loop -# -# python2.5 -m timeit -s 'import hashlib' -# 'h=hashlib.md5(); h.update("spork"); h.hexdigest()' -# ~4 usec per loop -# -# python2.5 -m timeit -s 'import hashlib;data = 1024 * "spork"' -# 'h=hashlib.new("md5"); h.update(data); h.hexdigest()' -# ~20 usec per loop -# -# python2.5 -m timeit -s 'import hashlib;data = 1024 * "spork"' -# 'h=hashlib.md5(); h.update(data); h.hexdigest()' -# ~18 usec per loop -# -# Summarized: -# - hashlib is faster than fchksum, fchksum is faster than python 2.4's md5. -# - using hashlib's new() instead of the predefined type is still noticably -# slower for 5k of data. Since ebuilds and patches will often be smaller -# than 5k we should avoid hashlib's new if there is a predefined type. -# - If we do not have hashlib preferring fchksum over python md5 is worth it. -# - Testing PyCrypto is unnecessary since its Crypto.Hash.MD5 is an -# alias for python's md5 (same for sha1). -# -# An additional advantage of using hashlib instead of PyCrypto is it -# is more reliable (PyCrypto has a history of generating bogus hashes, -# especially on non-x86 platforms, OpenSSL should be more reliable -# because it is more widely used). -# -# TODO do benchmarks for more hashes? -# -# Hash function we use is: -# - hashlib attr if available -# - hashlib through new() if available. -# - fchksum with python md5 fallback if possible -# - PyCrypto -# - python's md5 or sha1. - -chksum_types = {} - -try: - import hashlib -except ImportError: - pass -else: - # Always available according to docs.python.org: - # md5(), sha1(), sha224(), sha256(), sha384(), and sha512(). - for hashlibname, chksumname, size in [ - ('md5', 'md5', md5_size), - ('sha1', 'sha1', sha1_size), - ('sha256', 'sha256', sha256_size), - ]: - chksum_types[chksumname] = Chksummer(chksumname, - getattr(hashlib, hashlibname), size) - - # May or may not be available depending on openssl. List - # determined through trial and error. - for hashlibname, chksumname in [ - ('ripemd160', 'rmd160'), - ]: - try: - hashlib.new(hashlibname) - except ValueError: - pass # This hash is not available. - else: - chksum_types[chksumname] = Chksummer(chksumname, - partial(hashlib.new, hashlibname), rmd160_size) - del hashlibname, chksumname - - -if 'md5' not in chksum_types: - import md5 - fchksum = None - try: - import fchksum - except ImportError: - pass - else: - class MD5Chksummer(Chksummer): - chf_type = "md5" - str_size = md5_size - __init__ = lambda s:None - - def new(self): - return md5.new - - def __call__(self, filename): - if isinstance(filename, base_data_source): - if filename.get_path is not None: - filename = filename.get_path() - if isinstance(filename, basestring) and fchksum is not None: - return long(fchksum.fmd5t(filename)[0], 16) - return loop_over_file(filename, md5.new)[0] - - chksum_types = {"md5":MD5Chksummer()} - - -# expand this to load all available at some point -for k, v, str_size in (("sha1", "SHA", sha1_size), - ("sha256", "SHA256", sha256_size), - ("rmd160", "RIPEMD", rmd160_size)): - if k in chksum_types: - continue - try: - chksum_types[k] = Chksummer(k, modules.load_attribute( - "Crypto.Hash.%s.new" % v), str_size) - except modules.FailedImport: - pass -del k, v - - -for modulename, chksumname, size in [ - ('sha', 'sha1', sha1_size), - ('md5', 'md5', md5_size), - ]: - if chksumname not in chksum_types: - chksum_types[chksumname] = Chksummer(chksumname, - modules.load_attribute('%s.new' % (modulename,)), size) -del modulename, chksumname - -class SizeUpdater(object): - - def __init__(self): - self.count = 0 - - def update(self, data): - self.count += len(data) - - def hexdigest(self): - return "%x" % self.count - - -class SizeChksummer(Chksummer): - """ - size based chksum handler - yes, aware that size isn't much of a chksum. ;) - """ - - def __init__(self): - pass - obj = SizeUpdater - str_size = 1000000000 - chf_type = 'size' - - @staticmethod - def long2str(val): - return str(val) - - @staticmethod - def str2long(val): - return long(val) - - def __call__(self, file_obj): - if isinstance(file_obj, base_data_source): - if file_obj.get_path is not None: - file_obj = file_obj.get_path() - else: - file_obj = file_obj.get_fileobj() - if isinstance(file_obj, basestring): - try: - st_size = os.lstat(file_obj).st_size - except OSError: - return None - return st_size - # seek to the end. - file_obj.seek(0, 2) - return long(file_obj.tell()) - - -chksum_types["size"] = SizeChksummer() -chksum_types = dict((intern(k), v) for k, v in chksum_types.iteritems()) diff --git a/pkgcore/chksum/errors.py b/pkgcore/chksum/errors.py deleted file mode 100644 index a556be7..0000000 --- a/pkgcore/chksum/errors.py +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright: 2005 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -""" -exceptions thrown by chksum subsystem -""" - -class base(Exception): - pass - -class MissingChksum(base): - - def __init__(self, filename): - base.__init__(self, "Missing chksum file %r" % filename) - self.file = filename - - -class ParseChksumError(base): - def __init__(self, filename, error): - base.__init__(self, "Failed parsing %r chksum due to %s" % - (filename, error)) - self.file, self.error = filename, error diff --git a/pkgcore/chksum/gpg.py b/pkgcore/chksum/gpg.py deleted file mode 100644 index d80904f..0000000 --- a/pkgcore/chksum/gpg.py +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright: 2006 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -msg_header = "-----BEGIN PGP SIGNED MESSAGE-----\n" -msg_header_len = len(msg_header) -msg_hash = 'Hash:' -msg_hash_len = len(msg_hash) -sig_header = "-----BEGIN PGP SIGNATURE-----\n" -sig_header_len = len(sig_header) -sig_footer = "-----END PGP SIGNATURE-----\n" -sig_footer_len = len(sig_footer) - -def skip_signatures(iterable): - i = iter(iterable) -# format is- -#""" -#-----BEGIN PGP SIGNED MESSAGE----- -#Hash: SHA1 -# -#""" - - for line in i: - # so... prune msg first, then - if line.endswith(msg_header): - line = i.next() - while line[msg_hash_len:] == msg_hash: - line = i.next() - # skip blank line after msg. - i.next() - continue - while line.endswith(sig_header): - line = i.next() - # swallow the footer. - while not line.endswith(sig_footer): - line = i.next() - # leave the next line on the stack - line = i.next() - - yield line diff --git a/pkgcore/config/__init__.py b/pkgcore/config/__init__.py deleted file mode 100644 index 0261504..0000000 --- a/pkgcore/config/__init__.py +++ /dev/null @@ -1,80 +0,0 @@ -# Copyright: 2005 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -""" -configuration subsystem -""" - -# keep these imports as minimal as possible; access to -# pkgcore.config isn't uncommon, thus don't trigger till -# actually needed -from pkgcore.const import SYSTEM_CONF_FILE, USER_CONF_FILE - -class ConfigHint(object): - - """hint for introspection supplying overrides""" - - # be aware this is used in clone - __slots__ = ( - "types", "positional", "required", "typename", "allow_unknowns", "doc") - - def __init__(self, types=None, positional=None, required=None, doc=None, - typename=None, allow_unknowns=False): - self.types = types or {} - self.positional = positional or [] - self.required = required or [] - self.typename = typename - self.allow_unknowns = allow_unknowns - self.doc = doc - - def clone(self, **kwds): - new_kwds = {} - for attr in self.__slots__: - new_kwds[attr] = kwds.pop(attr, getattr(self, attr)) - if kwds: - raise TypeError("unknown type overrides: %r" % kwds) - return self.__class__(**new_kwds) - - -def configurable(*args, **kwargs): - """Decorator version of ConfigHint.""" - hint = ConfigHint(*args, **kwargs) - def decorator(original): - original.pkgcore_config_type = hint - return original - return decorator - - -def load_config(user_conf_file=USER_CONF_FILE, - system_conf_file=SYSTEM_CONF_FILE, - debug=False, prepend_sources=(), skip_config_files=False): - """ - the main entry point for any code looking to use pkgcore. - - @param user_conf_file: file to attempt to load, else defaults to trying to - load portage 2 style configs (/etc/make.conf, /etc/make.profile) - - @return: L{pkgcore.config.central.ConfigManager} instance - representing the system config. - """ - - from pkgcore.config import central, cparser - from pkgcore.plugin import get_plugins - import os - - configs = list(prepend_sources) - if not skip_config_files: - have_system_conf = os.path.isfile(system_conf_file) - have_user_conf = os.path.isfile(user_conf_file) - if have_system_conf or have_user_conf: - if have_user_conf: - configs.append(cparser.config_from_file(open(user_conf_file))) - if have_system_conf: - configs.append( - cparser.config_from_file(open(system_conf_file))) - else: - # make.conf... - from pkgcore.ebuild.portage_conf import config_from_make_conf - configs.append(config_from_make_conf()) - configs.extend(get_plugins('global_config')) - return central.ConfigManager(configs, debug=debug) diff --git a/pkgcore/config/basics.py b/pkgcore/config/basics.py deleted file mode 100644 index 1b96acd..0000000 --- a/pkgcore/config/basics.py +++ /dev/null @@ -1,536 +0,0 @@ -# Copyright: 2005 Marien Zwart <marienz@gentoo.org> -# License: GPL2 - - -""" -configuration subsystem primitives - -all callables can/may throw a -L{configuration exception<pkgcore.config.errors.ConfigurationError>} -""" - - -from pkgcore.config import errors, configurable -from snakeoil import currying -from snakeoil.demandload import demandload -demandload(globals(), "snakeoil:modules") - -type_names = ("list", "str", "bool", "int") - - -# Copied from inspect.py which copied it from compile.h. -# Also documented in http://docs.python.org/ref/types.html. -CO_VARARGS, CO_VARKEYWORDS = 4, 8 - -class ConfigType(object): - - """A configurable type. - - @ivar name: string specifying the protocol the instantiated object - conforms to. - @ivar callable: callable used to instantiate this type. - @ivar types: dict mapping key names to type strings. - @ivar positional: container holding positional arguments. - @ivar required: container holding required arguments. - @ivar allow_unknowns: controls whether unknown settings should error. - """ - - def __init__(self, func_obj): - """Create from a callable (function, member function, class). - - It uses the defaults to determine type: - - True or False mean it's a boolean - - a tuple means it's a list (of strings) - - a str means it's a string - - Arguments with a default of a different type are ignored. - - If an argument has no default, it is assumed to be a string- - exception to this is if the callable has a pkgcore_config_type - attr that is a L{ConfigHint} instance, in which case those - override. - """ - self.name = func_obj.__name__ - self.callable = func_obj - self.doc = getattr(func_obj, '__doc__', None) - if not hasattr(func_obj, 'func_code'): - # No function or method, should be a class so grab __init__. - func_obj = func_obj.__init__ - # We do not use the inspect module because that is a heavy - # thing to import and we can pretty easily get the data we - # need without it. Most of the code in its getargs function - # deals with tuples inside argument definitions, which we do - # not support anyway. - code = func_obj.func_code - args = code.co_varnames[:code.co_argcount] - varargs = bool(code.co_flags & CO_VARARGS) - varkw = bool(code.co_flags & CO_VARKEYWORDS) - if hasattr(func_obj, 'im_func'): - # It is a method. Chop off 'self': - args = args[1:] - defaults = func_obj.func_defaults - if defaults is None: - defaults = () - self.types = {} - # iterate through defaults backwards, so they match up to argnames - for i, default in enumerate(reversed(defaults)): - argname = args[-1 - i] - for typeobj, typename in [(bool, 'bool'), - (tuple, 'list'), - (str, 'str'), - ((int, long), 'int')]: - if isinstance(default, typeobj): - self.types[argname] = typename - break - # just [:-len(defaults)] doesn't work if there are no defaults - self.positional = args[:len(args)-len(defaults)] - # no defaults to determine the type from -> default to str. - for arg in self.positional: - self.types[arg] = 'str' - self.required = tuple(self.positional) - self.allow_unknowns = False - - # Process ConfigHint (if any) - hint_overrides = getattr(self.callable, "pkgcore_config_type", None) - if hint_overrides is not None: - self.types.update(hint_overrides.types) - if hint_overrides.required: - self.required = tuple(hint_overrides.required) - if hint_overrides.positional: - self.positional = tuple(hint_overrides.positional) - if hint_overrides.typename: - self.name = hint_overrides.typename - if hint_overrides.doc: - self.doc = hint_overrides.doc - self.allow_unknowns = hint_overrides.allow_unknowns - elif varargs or varkw: - raise TypeError( - 'func %s accepts *args or **kwargs, and no ConfigHint is ' - 'provided' % (self.callable,)) - - for var in ('class', 'inherit', 'default'): - if var in self.types: - raise errors.TypeDefinitionError( - '%s: you cannot change the type of %r' % ( - self.callable, var)) - - for var in self.positional: - if var not in self.required: - raise errors.TypeDefinitionError( - '%s: %r is in positionals but not in required' % - (self.callable, var)) - - -class LazySectionRef(object): - - """Abstract base class for lazy-loaded section references.""" - - def __init__(self, central, typename): - self.central = central - self.typename = typename.split(':', 1)[1] - self.cached_config = None - - def _collapse(self): - """Override this in a subclass.""" - raise NotImplementedError(self._collapse) - - def collapse(self): - """@returns: L{pkgcore.config.central.CollapsedConfig}.""" - if self.cached_config is None: - config = self.cached_config = self._collapse() - if self.typename is not None and config.type.name != self.typename: - raise errors.ConfigurationError( - 'reference should be of type %r, got %r' % ( - self.typename, config.type.name)) - return self.cached_config - - def instantiate(self): - """Convenience method returning the instantiated section.""" - return self.collapse().instantiate() - - -class LazyNamedSectionRef(LazySectionRef): - - def __init__(self, central, typename, name): - LazySectionRef.__init__(self, central, typename) - self.name = name - - def _collapse(self): - return self.central.collapse_named_section(self.name) - - -class LazyUnnamedSectionRef(LazySectionRef): - - def __init__(self, central, typename, section): - LazySectionRef.__init__(self, central, typename) - self.section = section - - def _collapse(self): - return self.central.collapse_section(self.section) - - -class ConfigSection(object): - - """ - Single Config section, returning typed values from a key. - - Not much of an object this, if we were using zope.interface it'd - be an Interface. - """ - - def __contains__(self, name): - """Check if a key is in this section.""" - raise NotImplementedError(self.__contains__) - - def keys(self): - """Return a list of keys.""" - raise NotImplementedError(self.keys) - - def get_value(self, central, name, arg_type): - """Return a setting, converted to the requested type.""" - raise NotImplementedError(self.get_value) - - -class DictConfigSection(ConfigSection): - - """Turns a dict and a conversion function into a ConfigSection.""" - - def __init__(self, conversion_func, source_dict): - """Initialize. - - @type conversion_func: callable. - @param conversion_func: called with a ConfigManager, a value from - the dict and a type name. - @type source_dict: dict with string keys and arbitrary values. - """ - ConfigSection.__init__(self) - self.func = conversion_func - self.dict = source_dict - - def __contains__(self, name): - return name in self.dict - - def keys(self): - return self.dict.keys() - - def get_value(self, central, name, arg_type): - try: - return self.func(central, self.dict[name], arg_type) - except errors.ConfigurationError, e: - e.stack.append('Converting argument %r to %s' % (name, arg_type)) - raise - - -class FakeIncrementalDictConfigSection(ConfigSection): - - """Turns a dict and a conversion function into a ConfigSection.""" - - def __init__(self, conversion_func, source_dict): - """Initialize. - - A request for a section of a list type will look for - name.prepend and name.append keys too, using those for values - prepended/appended to the inherited values. The conversion - func should return a single sequence for list types and in - repr for list types. - - @type conversion_func: callable. - @param conversion_func: called with a ConfigManager, a value from - the dict and a type name. - @type source_dict: dict with string keys and arbitrary values. - """ - ConfigSection.__init__(self) - self.func = conversion_func - self.dict = source_dict - - def __contains__(self, name): - return name in self.dict or name + '.append' in self.dict or \ - name + '.prepend' in self.dict - - def keys(self): - keys = set() - for key in self.dict: - if key.endswith('.append'): - keys.add(key[:-7]) - elif key.endswith('.prepend'): - keys.add(key[:-8]) - else: - keys.add(key) - return list(keys) - - def get_value(self, central, name, arg_type): - # Check if we need our special incremental magic. - if arg_type in ('list', 'str', 'repr') or arg_type.startswith('refs:'): - result = [] - # Careful: None is a valid dict value, so use something else here. - missing = object() - for subname in (name + '.prepend', name, name + '.append'): - val = self.dict.get(subname, missing) - if val is missing: - val = None - else: - try: - val = self.func(central, val, arg_type) - except errors.ConfigurationError, e: - e.stack.append('Converting argument %r to %s' % ( - subname, arg_type)) - raise - result.append(val) - if result[0] is result[1] is result[2] is None: - raise KeyError(name) - if arg_type != 'repr': - # Done. - return result - # If "kind" is of some incremental-ish kind or we have - # .prepend or .append for this key then we need to - # convert everything we have to the same kind and - # return all three. - # - # (we do not get called for separate reprs for the - # .prepend or .append because those are filtered from - # .keys(). If we do not filter those from .keys() - # central gets upset because it does not know their - # type. Perhaps this means we should have a separate - # .keys() used together with repr, not sure yet - # --marienz) - # - # The problem here is that we may get unsuitable for - # incremental or differing types for the three reprs - # we run, so we need to convert to a suitable common - # kind. - if result[0] is None and result[2] is None: - # Simple case: no extra data, so no need for any - # conversions. - kind, val = result[1] - if kind in ('list', 'str') or kind == 'refs': - # Caller expects a three-tuple. - return kind, (None, val, None) - else: - # non-incremental, just return as-is. - return kind, val - # We have more than one return value. Figure out what - # target to convert to. Choices are list, str and refs. - kinds = set(v[0] for v in result if v is not None) - if 'refs' in kinds or 'ref' in kinds: - # If we have any refs we have to convert to refs. - target_kind = 'refs' - elif kinds == set(['str']): - # If we have only str we can just use that. - target_kind = 'str' - else: - # Convert to list. May not make any sense, but is - # the best we can do. - target_kind = 'list' - converted = [] - for val in result: - if val is None: - converted.append(None) - continue - kind, val = val - if kind == 'ref': - assert target_kind == 'refs', target_kind - converted.append([val]) - elif kind == 'refs': - assert target_kind == 'refs', target_kind - converted.append(val) - elif kind == 'list': - assert target_kind != 'str', target_kind - converted.append(val) - else: - # Everything else gets converted to a string first. - if kind == 'callable': - val = '%s.%s' % (val.__module__, val.__name__) - elif kind in ('bool', 'int', 'str'): - val = str(val) - else: - raise errors.ConfigurationError( - 'unsupported type %r' % (kind,)) - # Then convert the str to list if needed. - if target_kind == 'str': - converted.append(val) - else: - converted.append([val]) - return target_kind, converted - # Not incremental. - try: - return self.func(central, self.dict[name], arg_type) - except errors.ConfigurationError, e: - e.stack.append('Converting argument %r to %s' % (name, arg_type)) - raise - -def convert_string(central, value, arg_type): - """Conversion func for a string-based DictConfigSection.""" - assert isinstance(value, basestring), value - if arg_type == 'callable': - try: - func = modules.load_attribute(value) - except modules.FailedImport: - raise errors.ConfigurationError('cannot import %r' % (value,)) - if not callable(func): - raise errors.ConfigurationError('%r is not callable' % (value,)) - return func - elif arg_type.startswith('refs:'): - return list(LazyNamedSectionRef(central, arg_type, ref) - for ref in list_parser(value)) - elif arg_type.startswith('ref:'): - return LazyNamedSectionRef(central, arg_type, str_parser(value)) - elif arg_type == 'repr': - return 'str', value - try: - func = { - 'list': list_parser, - 'str': str_parser, - 'bool': bool_parser, - 'int': int_parser, - }[arg_type] - except KeyError: - raise errors.ConfigurationError('Unknown type %r' % (arg_type,)) - return func(value) - -def convert_asis(central, value, arg_type): - """"Conversion" func assuming the types are already correct.""" - if arg_type == 'callable': - if not callable(value): - raise errors.ConfigurationError('%r is not callable' % (value,)) - return value - elif arg_type.startswith('ref:'): - if not isinstance(value, ConfigSection): - raise errors.ConfigurationError('%r is not a config section' % - (value,)) - return LazyUnnamedSectionRef(central, arg_type, value) - elif arg_type.startswith('refs:'): - l = [] - for section in value: - if not isinstance(section, ConfigSection): - raise errors.ConfigurationError('%r is not a config section' % - (value,)) - l.append(LazyUnnamedSectionRef(central, arg_type, section)) - return l - elif arg_type == 'repr': - if callable(value): - return 'callable', value - if isinstance(value, ConfigSection): - return 'ref', value - if isinstance(value, str): - return 'str', value - if isinstance(value, bool): - return 'bool', value - if isinstance(value, (list, tuple)): - if not value or isinstance(value[0], str): - return 'list', value - if isinstance(value[0], ConfigSection): - return 'refs', value - raise errors.ConfigurationError('unsupported type for %r' % (value,)) - elif not isinstance(value, {'list': (list, tuple), - 'str': str, - 'bool': bool}[arg_type]): - raise errors.ConfigurationError( - '%r does not have type %r' % (value, arg_type)) - return value - -def convert_hybrid(central, value, arg_type): - """Automagically switch between L{convert_string} and L{convert_asis}. - - L{convert_asis} is used for arg_type str and if value is not a basestring. - L{convert_string} is used for the rest. - - Be careful about handing in escaped strings: they are not - unescaped (for arg_type str). - """ - if arg_type != 'str' and isinstance(value, basestring): - return convert_string(central, value, arg_type) - return convert_asis(central, value, arg_type) - -# "Invalid name" (pylint thinks these are module-level constants) -# pylint: disable-msg=C0103 -HardCodedConfigSection = currying.partial( - FakeIncrementalDictConfigSection, convert_asis) -ConfigSectionFromStringDict = currying.partial( - FakeIncrementalDictConfigSection, convert_string) -AutoConfigSection = currying.partial( - FakeIncrementalDictConfigSection, convert_hybrid) - - -def section_alias(target, typename): - """Build a ConfigSection that instantiates a named reference. - - Because of central's caching our instantiated value will be - identical to our target's. - """ - @configurable({'target': 'ref:' + typename}, typename=typename) - def alias(target): - return target - return AutoConfigSection({'class': alias, 'target': target}) - - -def list_parser(string): - """split on whitespace honoring quoting for new tokens""" - l = [] - i = 0 - e = len(string) - # check for stringness because we return something interesting if - # feeded a sequence of strings - if not isinstance(string, basestring): - raise TypeError('expected a string, got %r' % (string,)) - while i < e: - if not string[i].isspace(): - if string[i] in ("'", '"'): - q = i - i += 1 - res = [] - while i < e and string[i] != string[q]: - if string[i] == '\\': - i += 1 - res.append(string[i]) - i += 1 - if i >= e: - raise errors.QuoteInterpretationError(string) - l.append(''.join(res)) - else: - res = [] - while i < e and not (string[i].isspace() or - string[i] in ("'", '"')): - if string[i] == '\\': - i += 1 - res.append(string[i]) - i += 1 - if i < e and string[i] in ("'", '"'): - raise errors.QuoteInterpretationError(string) - l.append(''.join(res)) - i += 1 - return l - -def str_parser(string): - """yank leading/trailing whitespace and quotation, along with newlines""" - s = string.strip() - if len(s) > 1 and s[0] in '"\'' and s[0] == s[-1]: - s = s[1:-1] - return s.replace('\n', ' ').replace('\t', ' ') - -def bool_parser(string): - """convert a string to a boolean""" - s = str_parser(string).lower() - if s in ("no", "false", "0"): - return False - if s in ("yes", "true", "1"): - return True - raise errors.ConfigurationError('%r is not a boolean' % s) - -def int_parser(string): - """convert a string to a integer""" - string = str_parser(string) - try: - return int(string) - except ValueError: - raise errors.ConfigurationError('%r is not an integer' % string) - -@configurable({'path': 'str', 'parser': 'callable'}, typename='configsection') -def parse_config_file(path, parser): - try: - f = open(path, 'r') - except (IOError, OSError), e: - raise errors.InstantiationError(e.strerror) - try: - return parser(f) - finally: - f.close() diff --git a/pkgcore/config/central.py b/pkgcore/config/central.py deleted file mode 100644 index 87f3941..0000000 --- a/pkgcore/config/central.py +++ /dev/null @@ -1,490 +0,0 @@ -# Copyright: 2005-2006 Marien Zwart <marienz@gentoo.org> -# Copyright: 2005-2006 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -"""Collapse multiple config-sources and instantiate from them. - -A lot of extra documentation on this is in dev-notes/config.rst. -""" - - -from pkgcore.config import errors, basics -from snakeoil import mappings - - -class _ConfigMapping(mappings.DictMixin): - - """Minimal dict-like wrapper returning config sections by type. - - Similar to L{LazyValDict<mappings.LazyValDict>} but __getitem__ - does not call the key func for __getitem__. - - Careful: getting the keys for this mapping will collapse all of - central's configs to get at their types, which might be slow if - any of them are remote! - """ - - def __init__(self, manager, typename): - mappings.DictMixin.__init__(self) - self.manager = manager - self.typename = typename - - def __getitem__(self, key): - conf = self.manager.collapse_named_section(key, raise_on_missing=False) - if conf is None or conf.type.name != self.typename: - raise KeyError(key) - try: - return conf.instantiate() - except errors.ConfigurationError, e: - e.stack.append('Instantiating named section %r' % (key,)) - raise - - def iterkeys(self): - for config in self.manager.configs: - for name in config: - try: - collapsed = self.manager.collapse_named_section(name) - except errors.BaseException: - # Cannot be collapsed, ignore it (this is not - # an error, it can be used as base for - # something that can be collapsed) - pass - else: - if collapsed.type.name == self.typename: - yield name - - def __contains__(self, key): - conf = self.manager.collapse_named_section(key, raise_on_missing=False) - return conf is not None and conf.type.name == self.typename - - -class CollapsedConfig(object): - - """A collapsed config section. - - @type type: L{basics.ConfigType} - @ivar type: Our type. - @type config: dict - @ivar config: The supplied configuration values. - @ivar debug: if True exception wrapping is disabled. - @ivar default: True if this section is a default. - @type name: C{str} or C{None} - @ivar name: our section name or C{None} for an anonymous section. - """ - - def __init__(self, type_obj, config, manager, debug=False, default=False): - """Initialize instance vars.""" - # Check if we got all values required to instantiate. - missing = set(type_obj.required) - set(config) - if missing: - raise errors.ConfigurationError( - 'type %s.%s needs settings for %s' % ( - type_obj.callable.__module__, - type_obj.callable.__name__, - ', '.join(repr(var) for var in missing))) - - self.name = None - self.default = default - self.debug = debug - self.type = type_obj - self.config = config - # Cached instance if we have one. - self._instance = None - - def instantiate(self): - """Call our type's callable, cache and return the result. - - Calling instantiate more than once will return the cached value. - """ - if self._instance is not None: - return self._instance - - # Needed because this code can run twice even with instance - # caching if we trigger an InstantiationError. - config = mappings.ProtectedDict(self.config) - - # Instantiate section refs. - # Careful: not everything we have for needs to be in the conf dict - # (because of default values) and not everything in the conf dict - # needs to have a type (because of allow_unknowns). - for name, val in config.iteritems(): - typename = self.type.types.get(name) - if typename is None: - continue - # central already checked the type, no need to repeat that here. - if typename.startswith('ref:'): - try: - config[name] = val.instantiate() - except errors.ConfigurationError, e: - e.stack.append('Instantiating ref %r' % (name,)) - raise - elif typename.startswith('refs:'): - try: - config[name] = list(ref.instantiate() for ref in val) - except errors.ConfigurationError, e: - e.stack.append('Instantiating refs %r' % (name,)) - raise - - callable_obj = self.type.callable - - pargs = [] - for var in self.type.positional: - pargs.append(config.pop(var)) - # Python is basically the worst language ever: - # TypeError: repo() argument after ** must be a dictionary - configdict = dict(config) - try: - self._instance = callable_obj(*pargs, **configdict) - except errors.InstantiationError, e: - # This is probably just paranoia, but better safe than sorry. - if e.callable is None: - e.callable = callable_obj - e.pargs = pargs - e.kwargs = configdict - raise - except (RuntimeError, SystemExit, KeyboardInterrupt): - raise - except Exception, e: - if self.debug: - raise - raise errors.InstantiationError(exception=e, - callable_obj=callable_obj, - pargs=pargs, kwargs=configdict) - if self._instance is None: - raise errors.InstantiationError( - 'No object returned', callable_obj=callable_obj, pargs=pargs, - kwargs=configdict) - - return self._instance - - -class ConfigManager(object): - - """Combine config type definitions and configuration sections. - - Creates instances of a requested type and name by pulling the - required data from any number of provided configuration sources. - - The following special type names are recognized: - - configsection: instantiated and used the same way as an entry in the - configs L{__init__} arg. - - remoteconfigsection: Instantiated and used the same way as an entry in - theremote_configs L{__init__} arg. - - These "magic" typenames are only recognized if they are used by a - section with a name starting with "autoload". - """ - - def __init__(self, configs=(), remote_configs=(), debug=False): - """Initialize. - - @type configs: sequence of mappings of string to ConfigSection. - @param configs: configuration to use. - Can define extra configs that are also loaded. - @type remote_configs: sequence of mappings of string to ConfigSection. - @param remote_configs: configuration to use. - Cannot define extra configs. - @param debug: if set to True exception wrapping is disabled. - This means things can raise other exceptions than - ConfigurationError but tracebacks are complete. - """ - self.original_configs = tuple(configs) - self.original_remote_configs = tuple(remote_configs) - # Set of encountered section names, used to catch recursive references. - self._refs = set() - self.debug = debug - self.reload() - - def reload(self): - """Reinitialize us from the config sources originally passed in. - - This throws away all cached instances and re-executes autoloads. - """ - # "Attribute defined outside __init__" - # pylint: disable-msg=W0201 - self.configs = (list(self.original_configs) + - list(self.original_remote_configs)) - # Cache mapping confname to CollapsedConfig. - self.collapsed_configs = {} - self._exec_configs(self.original_configs) - - __getattr__ = _ConfigMapping - - def _exec_configs(self, configs): - """Pull extra type and config sections from configs and use them. - - Things loaded this way are added after already loaded things - (meaning the config containing the autoload section overrides - the config(s) added by that section). - """ - new_configs = [] - for config in configs: - for name in config: - # Do not even touch the ConfigSection if it's not an autoload. - if not name.startswith('autoload'): - continue - # If this matches something we previously instantiated - # we should probably blow up to prevent massive - # amounts of confusion (and recursive autoloads) - if name in self.collapsed_configs: - raise errors.ConfigurationError( - 'section %r from autoload is already collapsed!' % ( - name,)) - try: - collapsed = self.collapse_named_section(name) - except errors.ConfigurationError, e: - e.stack.append('Collapsing autoload %r' % (name,)) - raise - if collapsed.type.name not in ( - 'configsection', 'remoteconfigsection'): - raise errors.ConfigurationError( - 'Section %r is marked as autoload but type is %s, not ' - '(remote)configsection' % (name, collapsed.type.name)) - try: - instance = collapsed.instantiate() - except errors.ConfigurationError, e: - e.stack.append('Instantiating autoload %r' % (name,)) - raise - if collapsed.type.name == 'configsection': - new_configs.append(instance) - elif collapsed.type.name == 'remoteconfigsection': - self.configs.append(instance) - if new_configs: - self.configs.extend(new_configs) - self._exec_configs(new_configs) - - def sections(self): - """Return an iterator of all section names.""" - for config in self.configs: - for name in config: - yield name - - def collapse_named_section(self, name, raise_on_missing=True): - """Collapse a config by name, possibly returning a cached instance. - - @returns: L{CollapsedConfig}. - - If there is no section with this name a ConfigurationError is raised, - unless raise_on_missing is False in which case None is returned. - """ - if name in self._refs: - raise errors.ConfigurationError( - 'Reference to %r is recursive' % (name,)) - self._refs.add(name) - try: - result = self.collapsed_configs.get(name) - if result is not None: - return result - for source_index, config in enumerate(self.configs): - if name in config: - section = config[name] - break - else: - if raise_on_missing: - raise errors.ConfigurationError( - 'no section called %r' % (name,)) - return None - try: - result = self.collapse_section(section, name, source_index) - result.name = name - except errors.ConfigurationError, e: - e.stack.append('Collapsing section named %r' % (name,)) - raise - self.collapsed_configs[name] = result - return result - finally: - self._refs.remove(name) - - def collapse_section(self, section, _name=None, _index=None): - """Collapse a ConfigSection to a L{CollapsedConfig}.""" - - # Bail if this is an inherit-only (uncollapsable) section. - try: - inherit_only = section.get_value(self, 'inherit-only', 'bool') - except KeyError: - pass - else: - if inherit_only: - raise errors.CollapseInheritOnly( - 'cannot collapse inherit-only section') - - # List of (name, ConfigSection, index) tuples, most specific first. - slist = [(_name, section, _index)] - - # first map out inherits. - inherit_names = set([_name]) - for current_section, current_conf, index in slist: - if 'inherit' not in current_conf: - continue - prepend, inherits, append = current_conf.get_value( - self, 'inherit', 'list') - if prepend is not None or append is not None: - raise errors.ConfigurationError( - 'Prepending or appending to the inherit list makes no ' - 'sense') - for inherit in inherits: - if inherit == current_section: - # Self-inherit is a bit special. - for i, config in enumerate(self.configs[index + 1:]): - if inherit in config: - slist.append((inherit, config[inherit], - index + i + 1)) - break - else: - raise errors.ConfigurationError( - 'Self-inherit %r cannot be found' % (inherit,)) - else: - if inherit in inherit_names: - raise errors.ConfigurationError( - 'Inherit %r is recursive' % (inherit,)) - inherit_names.add(inherit) - for i, config in enumerate(self.configs): - if inherit in config: - slist.append((inherit, config[inherit], i)) - break - else: - raise errors.ConfigurationError( - 'inherit target %r cannot be found' % (inherit,)) - - # Grab the "class" setting first (we need it to get a type obj - # to collapse to the right type in the more general loop) - for inherit_name, inherit_conf, index in slist: - if "class" in inherit_conf: - break - else: - raise errors.ConfigurationError('no class specified') - - type_obj = basics.ConfigType(inherit_conf.get_value(self, 'class', - 'callable')) - - conf = {} - for section_nr, (inherit_name, inherit_conf, index) in \ - enumerate(reversed(slist)): - for key in inherit_conf.keys(): - if key in ('class', 'inherit', 'inherit-only'): - continue - typename = type_obj.types.get(key) - if typename is None: - if key == 'default': - typename = 'bool' - elif not type_obj.allow_unknowns: - if section_nr != len(slist) - 1: - raise errors.ConfigurationError( - 'Type of %r inherited from %r unknown' % ( - key, inherit_name)) - raise errors.ConfigurationError( - 'Type of %r unknown' % (key,)) - else: - typename = 'str' - is_ref = typename.startswith('ref:') - is_refs = typename.startswith('refs:') - # The sections do not care about lazy vs nonlazy. - if typename.startswith('lazy_'): - typename = typename[5:] - result = inherit_conf.get_value(self, key, typename) - if is_ref: - try: - result = result.collapse() - except errors.ConfigurationError, e: - e.stack.append( - 'Collapsing section ref %r' % (key,)) - raise - elif is_refs: - try: - result = list( - list(ref.collapse() for ref in subresult or ()) - for subresult in result) - except errors.ConfigurationError, e: - e.stack.append( - 'Collapsing section refs %r' % (key,)) - raise - if typename == 'list' or typename.startswith('refs:'): - prepend, result, append = result - if result is None: - if key in conf: - result = conf[key] - else: - result = [] - if prepend: - result = prepend + result - if append: - result += append - elif typename == 'str': - prepend, result, append = result - if result is None and key in conf: - result = conf[key] - result = ' '.join( - v for v in (prepend, result, append) if v) - conf[key] = result - default = conf.pop('default', False) - return CollapsedConfig( - type_obj, conf, self, debug=self.debug, default=default) - - def get_default(self, type_name): - """Finds the configuration specified default obj of type_name. - - Returns C{None} if no defaults. - """ - # The name of the "winning" default or None if there is none. - default_name = None - # The collapsed default section or None. - default = None - for source in self.configs: - for name, section in source.iteritems(): - collapsed = None - try: - is_default = section.get_value(self, 'default', 'bool') - except KeyError: - is_default = False - if not is_default: - continue - # We need to know the type name of this section, for - # which we need the class. Try to grab this from the - # section directly: - try: - klass = section.get_value(self, 'class', 'callable') - except errors.ConfigurationError: - # There is a class setting but it is not valid. - # This means it is definitely not the one we are - # interested in, so just skip this. - continue - except KeyError: - # There is no class value on the section. Collapse - # it to see if it inherits one: - try: - collapsed = self.collapse_named_section(name) - except errors.ConfigurationError: - # Uncollapsable. Just ignore this, since we - # have no clean way of determining if this - # would be an "interesting" section if it - # could be collapsed (and complaining about - # every uncollapsable section with - # default=true would be too much). - continue - type_obj = collapsed.type - else: - # Grabbed the class directly from the section. - type_obj = basics.ConfigType(klass) - if type_obj.name != type_name: - continue - if default_name is not None: - raise errors.ConfigurationError( - 'both %r and %r are default for %r' % ( - default_name, name, type_name)) - default_name = name - default = collapsed - if default_name is not None: - if default is None: - try: - default = self.collapse_named_section(default_name) - except errors.ConfigurationError, e: - e.stack.append('Collapsing default %s %r' % ( - type_name, default_name)) - raise - try: - return default.instantiate() - except errors.ConfigurationError, e: - e.stack.append('Instantiating default %s %r' % - (type_name, default_name)) - raise - return None diff --git a/pkgcore/config/cparser.py b/pkgcore/config/cparser.py deleted file mode 100644 index 5ba5817..0000000 --- a/pkgcore/config/cparser.py +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright: 2005 Marien Zwart <marienz@gentoo.org> -# License: GPL2 - -""" -ini based configuration format -""" - -from ConfigParser import ConfigParser - -from pkgcore.config import basics -from snakeoil import mappings - -class CaseSensitiveConfigParser(ConfigParser): - def optionxform(self, val): - return val - - -def config_from_file(file_obj): - """ - generate a config dict - - @param file_obj: file protocol instance - @return: L{snakeoil.mappings.LazyValDict} instance - """ - cparser = CaseSensitiveConfigParser() - cparser.readfp(file_obj) - def get_section(section): - return basics.ConfigSectionFromStringDict(dict(cparser.items(section))) - return mappings.LazyValDict(cparser.sections, get_section) diff --git a/pkgcore/config/dhcpformat.py b/pkgcore/config/dhcpformat.py deleted file mode 100644 index 357596f..0000000 --- a/pkgcore/config/dhcpformat.py +++ /dev/null @@ -1,169 +0,0 @@ -# Copyright: 2005 Marien Zwart <marienz@gentoo.org> -# License: GPL2 - -"""Parse a dhcpd.conf(5) style configuration file. - -Example of the supported format (not a complete config):: - - # this is a comment. - # this is a config section. - metadata { - # strings may be quoted or unquoted. semicolon terminator. - type cache; - class pkgcore.cache.metadata.database; - # true, yes or 1 for boolean True (case-insensitive). - readonly true; - location /usr/portage/ - } - - # this needs to be quoted because it has a space in it - "livefs domain" { - # this could be unquoted. - type "domain"; - package.keywords "/etc/portage/package.keywords"; - default yes; - # this is a section reference, with a nested anonymous section. - repositories { - type repo; - class pkgcore.ebuild.repository.tree; - location /usr/portage; - # this is also a section reference, but instead of a nested section - # we refer to the named metadata section above - cache metadata; - }; - fetcher { - type fetcher; - distdir /usr/portage/distfiles; - } - } -""" - -from pkgcore.config import basics, errors -from snakeoil import mappings, modules, demandload -demandload.demandload(globals(), 'snakeoil.compatibility:all') - -import pyparsing as pyp - -# this is based on the 'BIND named.conf parser' on pyparsing's webpage - -_section = pyp.Forward() -_value = (pyp.Word(pyp.alphanums + './_') | - pyp.quotedString.copy().setParseAction(pyp.removeQuotes)) - -_section_contents = pyp.dictOf( - _value, pyp.Group(pyp.OneOrMore(_value | _section)) + pyp.Suppress(';')) - -# "statement seems to have no effect" -# pylint: disable-msg=W0104 -_section << pyp.Group(pyp.Suppress('{') + _section_contents + - pyp.Suppress('}')) - -parser = ( - pyp.stringStart + - pyp.dictOf(_value, _section).ignore(pyp.pythonStyleComment) + - pyp.stringEnd) - - -class ConfigSection(basics.ConfigSection): - - """Expose a section_contents from pyparsing as a ConfigSection. - - mke2fsformat also uses this. - """ - - def __init__(self, section): - basics.ConfigSection.__init__(self) - self.section = section - - def __contains__(self, name): - return name in self.section - - def keys(self): - return self.section.keys() - - def get_value(self, central, name, arg_type): - value = self.section[name] - if arg_type == 'callable': - if len(value) != 1: - raise errors.ConfigurationError('only one argument required') - value = value[0] - if not isinstance(value, basestring): - raise errors.ConfigurationError( - 'need a callable, not a section') - try: - value = modules.load_attribute(value) - except modules.FailedImport: - raise errors.ConfigurationError('cannot import %r' % (value,)) - if not callable(value): - raise errors.ConfigurationError('%r is not callable' % value) - return value - elif arg_type.startswith('ref:'): - if len(value) != 1: - raise errors.ConfigurationError('only one argument required') - value = value[0] - if isinstance(value, basestring): - # it's a section ref - return basics.LazyNamedSectionRef(central, arg_type, value) - else: - # it's an anonymous inline section - return basics.LazyUnnamedSectionRef(central, arg_type, - ConfigSection(value)) - elif arg_type.startswith('refs:'): - result = [] - for ref in value: - if isinstance(ref, basestring): - # it's a section ref - result.append(basics.LazyNamedSectionRef( - central, arg_type, ref)) - else: - # it's an anonymous inline section - result.append(basics.LazyUnnamedSectionRef( - central, arg_type, ConfigSection(ref))) - return result - elif arg_type == 'list': - if not isinstance(value, basestring): - # sequence - value = ' '.join(value) - return None, basics.list_parser(value), None - elif arg_type == 'repr': - if len(value) == 1: - value = value[0] - if isinstance(value, basestring): - return 'str', value - else: - return 'ref', ConfigSection(value) - else: - if all(isinstance(v, basestring) for v in value): - return 'list', list(value) - result = [] - for v in value: - if isinstance(v, basestring): - result.append(v) - else: - result.append(ConfigSection(v)) - return 'refs', result - assert False, 'unreachable' - else: - if len(value) != 1: - raise errors.ConfigurationError('only one argument required') - if not isinstance(value[0], basestring): - raise errors.ConfigurationError( - '%r should be a string' % value) - if arg_type == 'str': - return [None, basics.str_parser(value[0]), None] - elif arg_type == 'bool': - return basics.bool_parser(value[0]) - else: - raise errors.ConfigurationError( - 'unsupported type %r' % (arg_type,)) - - -def config_from_file(file_obj): - try: - config = parser.parseFile(file_obj) - except pyp.ParseException, e: - name = getattr(file_obj, 'name', file_obj) - raise errors.ConfigurationError('%s: %s' % (name, e)) - def build_section(name): - return ConfigSection(config[name]) - return mappings.LazyValDict(config.keys, build_section) diff --git a/pkgcore/config/domain.py b/pkgcore/config/domain.py deleted file mode 100644 index f9c13df..0000000 --- a/pkgcore/config/domain.py +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright: 2006 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -""" -base class to derive from for domain objects - -Bit empty at the moment -""" -from snakeoil.demandload import demandload -demandload(globals(), "pkgcore.repository:multiplex") - -# yes this is basically empty. will fill it out as the base is better -# identified. - -class domain(object): - - def __getattr__(self, attr): - if attr == "all_repos": - if len(self.repos) == 1: - a = self.all_repos = self.repos[0] - else: - a = self.all_repos = multiplex.tree(*self.repos) - elif attr == "all_vdbs": - if len(self.vdb) == 1: - a = self.all_vdbs = self.vdb[0] - else: - a = self.all_vdbs = multiplex.tree(*self.vdb) - else: - raise AttributeError(attr) - return a diff --git a/pkgcore/config/errors.py b/pkgcore/config/errors.py deleted file mode 100644 index 9dd0e2a..0000000 --- a/pkgcore/config/errors.py +++ /dev/null @@ -1,104 +0,0 @@ -# Copyright: 2005 Marien Zwart <marienz@gentoo.org> -# Copyright: 2005 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -# potentially use an intermediate base for user config errors, -# seperate base for instantiation? - - -"""Exceptions raised by the config code.""" - - -class BaseException(Exception): - pass - - -class TypeDefinitionError(BaseException): - """Fatal error in type construction.""" - - -class ConfigurationError(BaseException): - - """Fatal error in parsing a config section. - - @type stack: sequence of strings. - @ivar stack: messages describing where this ConfigurationError originated. - configuration-related code catching ConfigurationError that wants to - raise its own ConfigurationError should modify (usually append to) - the stack and then re-raise the original exception (this makes sure - the traceback is preserved). - """ - - def __init__(self, message): - BaseException.__init__(self, message) - self.stack = [message] - - def __str__(self): - return ':\n'.join(reversed(self.stack)) - - -class CollapseInheritOnly(ConfigurationError): - """Attempt was made to collapse an uncollapsable section. - - Separate exception because pconfig catches it separately. - """ - - -class InstantiationError(ConfigurationError): - - """Exception occured during instantiation. - - @ivar callable: callable object which failed during instantiation. - @ivar pargs: positional args passed to callable. - @ivar kwargs: keyword args passed to callable. - @ivar exc: Original exception object or None. - - A well-behaved configurable callable should raise this exception - if instantiation failed, providing one or both of message and - exception. The other fields will be filled in by central. - - If the callable raises something else central will wrap it in - this, but that will lose the traceback. - """ - - def __init__(self, message=None, exception=None, callable_obj=None, - pargs=None, kwargs=None): - if message is not None: - ConfigurationError.__init__(self, message) - elif exception is not None: - ConfigurationError.__init__(self, str(exception)) - else: - raise ValueError('specify at least one of message and exception') - self.message = message - self.callable = callable_obj - self.pargs = pargs - self.kwargs = kwargs - self.exc = exception - - def __str__(self): - # self.callable should be set here (nothing should try to catch - # and str() this before central had a chance to fill it in) - if self.message is not None: - if self.callable is None: - message = '%r, callable unset!' % (self.message,) - else: - message = '%r instantiating %s.%s' % ( - self.message, self.callable.__module__, - self.callable.__name__) - # The weird repr(str(exc)) used here quotes the message nicely. - elif self.callable is not None: - message = "Caught exception %r instantiating %s.%s" % ( - str(self.exc), self.callable.__module__, - self.callable.__name__) - else: - message = "Caught exception %r, callable unset!" % (str(self.exc),) - return ':\n'.join(reversed([message] + self.stack[1:])) - - -class QuoteInterpretationError(ConfigurationError): - - """Quoting of a var was screwed up.""" - - def __init__(self, string): - ConfigurationError.__init__(self, "Parsing of %r failed" % (string,)) - self.str = string diff --git a/pkgcore/config/mke2fsformat.py b/pkgcore/config/mke2fsformat.py deleted file mode 100644 index 212d597..0000000 --- a/pkgcore/config/mke2fsformat.py +++ /dev/null @@ -1,77 +0,0 @@ -# Copyright: 2005 Marien Zwart <marienz@gentoo.org> -# License: GPL2 - -"""Parse a mke2fs.conf(5) style configuration file. - -Example of the supported format (not a complete config):: - - # this is a comment. - # this is a config section. - [metadata] - # strings may be quoted or unquoted. semicolon terminator. - type = cache - class = pkgcore.cache.metadata.database - # true, yes or 1 for boolean True (case-insensitive). - readonly = true - location = /usr/portage/ - - - # this needs to be quoted because it has a space in it - [livefs domain] - # this could be unquoted. - type = "domain" - package.keywords = "/etc/portage/package.keywords" - default = yes - # this is a section reference, with a nested anonymous section. - repositories = { - type = repo - class = pkgcore.ebuild.repository.tree - location = /usr/portage - # this is also a section reference, but instead of a nested section - # we refer to the named metadata section above - cache = metadata - } - fetcher = { - type = fetcher - distdir = /usr/portage/distfiles - } -""" - -# The tests for this are in test_dhcpformat. - -from pkgcore.config import dhcpformat, errors -from snakeoil import mappings -import pyparsing as pyp - - -_section_contents = pyp.Forward() -_value = (pyp.Word(pyp.alphanums + './_').setWhitespaceChars(' \t') | - pyp.quotedString.copy().setParseAction(pyp.removeQuotes)) - -_section = pyp.Group( - pyp.Suppress('{' + pyp.lineEnd) + _section_contents + pyp.Suppress('}')) - -# "statement seems to have no effect" -# pylint: disable-msg=W0104 -_section_contents << pyp.dictOf( - _value + pyp.Suppress('='), - pyp.Group(pyp.OneOrMore((_value | _section).setWhitespaceChars(' \t'))) + - pyp.Suppress(pyp.lineEnd)) - -parser = ( - pyp.stringStart + - pyp.dictOf( - pyp.Suppress('[') + _value + pyp.Suppress(']' + pyp.lineEnd), - _section_contents).ignore(pyp.pythonStyleComment) + - pyp.stringEnd) - - -def config_from_file(file_obj): - try: - config = parser.parseFile(file_obj) - except pyp.ParseException, e: - name = getattr(file_obj, 'name', file_obj) - raise errors.ConfigurationError('%s: %s' % (name, e)) - def build_section(name): - return dhcpformat.ConfigSection(config[name]) - return mappings.LazyValDict(config.keys, build_section) diff --git a/pkgcore/const.py b/pkgcore/const.py deleted file mode 100644 index c92ea49..0000000 --- a/pkgcore/const.py +++ /dev/null @@ -1,65 +0,0 @@ -# Copyright: 2005-2007 Brian Harring <ferringb@gmail.com> -# Copyright: 2000-2005 Gentoo Foundation -# License: GPL2 - - -""" -Internal constants. - -Future of this module is debatable- for the most part you likely don't -want to be using this. As soon as is possible, most of these defines -will be shifted to wherever they're best situated. -""" - -# note this is lifted out of portage 2. so... it's held onto for the -# sake of having stuff we still need, but it does need cleanup. - -import os.path as osp - - -# the pkgcore package directory -PORTAGE_BASE_PATH = osp.dirname(osp.abspath(__file__)) -PKGCORE_BIN_PATH = osp.join(PORTAGE_BASE_PATH, 'bin') -SYSTEM_CONF_FILE = '/etc/pkgcore.conf' -USER_CONF_FILE = osp.expanduser('~/.pkgcore.conf') - -#PORTAGE_PYM_PATH = PORTAGE_BASE_PATH+"/pym" -#PROFILE_PATH = "/etc/make.profile" -LOCALE_DATA_PATH = PORTAGE_BASE_PATH+"/locale" - -EBUILD_DAEMON_PATH = PKGCORE_BIN_PATH+"/ebuild-env/ebuild-daemon.sh" - -SANDBOX_BINARY = "/usr/bin/sandbox" - -DEPSCAN_SH_BINARY = "/sbin/depscan.sh" -BASH_BINARY = "/bin/bash" -MOVE_BINARY = "/bin/mv" -COPY_BINARY = "/bin/cp" -PRELINK_BINARY = "/usr/sbin/prelink" -depends_phase_path = PKGCORE_BIN_PATH+"/ebuild-env/:/bin:/usr/bin" -EBUILD_ENV_PATH = [PKGCORE_BIN_PATH+"/"+x for x in [ - "ebuild-env", "ebuild-helpers"]] \ - + ["/sbin", "/bin", "/usr/sbin", "/usr/bin"] -EBD_ENV_PATH = PKGCORE_BIN_PATH+"/ebuild-env" - -# XXX this is out of place -WORLD_FILE = '/var/lib/portage/world' -#MAKE_CONF_FILE = "/etc/make.conf" -#MAKE_DEFAULTS_FILE = PROFILE_PATH + "/make.defaults" - -# XXX this is out of place -CUSTOM_MIRRORS_FILE = "/etc/portage/mirrors" -SANDBOX_PIDS_FILE = "/tmp/sandboxpids.tmp" - -#CONFCACHE_FILE = CACHE_PATH+"/confcache" -#CONFCACHE_LIST = CACHE_PATH+"/confcache_files.anydbm" - -LIBFAKEROOT_PATH = "/usr/lib/libfakeroot.so" -FAKED_PATH = "/usr/bin/faked" - -RSYNC_BIN = "/usr/bin/rsync" -RSYNC_HOST = "rsync.gentoo.org/gentoo-portage" - -CVS_BIN = "/usr/bin/cvs" - -VERSION = '0.3.1' diff --git a/pkgcore/ebuild/__init__.py b/pkgcore/ebuild/__init__.py deleted file mode 100644 index c3c11f3..0000000 --- a/pkgcore/ebuild/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright: 2005 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -""" -gentoo ebuild support -""" diff --git a/pkgcore/ebuild/atom.py b/pkgcore/ebuild/atom.py deleted file mode 100644 index ca56898..0000000 --- a/pkgcore/ebuild/atom.py +++ /dev/null @@ -1,504 +0,0 @@ -# Copyright: 2005-2007 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -# "More than one statement on a single line" -# pylint: disable-msg=C0321 - -""" -gentoo ebuild atom, should be generalized into an agnostic base -""" - -from pkgcore.restrictions import values, packages, boolean -from pkgcore.ebuild import cpv, errors -from pkgcore.ebuild.atom_restricts import VersionMatch -from snakeoil.compatibility import all -from snakeoil.klass import generic_equality -from snakeoil.demandload import demandload -demandload(globals(), - "pkgcore.restrictions.delegated:delegate", - "snakeoil.currying:partial", -) - -# namespace compatibility... -MalformedAtom = errors.MalformedAtom - -valid_use_chars = set(str(x) for x in xrange(10)) -valid_use_chars.update(chr(x) for x in xrange(ord("a"), ord("z"))) -valid_use_chars.update(chr(x) for x in xrange(ord("A"), ord("Z"))) -valid_use_chars.update(["_", ".", "+", "-"]) -valid_use_chars = frozenset(valid_use_chars) - -def native_init(self, atom, negate_vers=False): - """ - @param atom: string, see gentoo ebuild atom syntax - """ - sf = object.__setattr__ - - orig_atom = atom - - u = atom.find("[") - if u != -1: - # use dep - u2 = atom.find("]", u) - if u2 == -1: - raise errors.MalformedAtom(atom, - "use restriction isn't completed") - sf(self, "use", tuple(sorted(atom[u+1:u2].split(',')))) - for x in self.use: - if not all(y in valid_use_chars for y in x): - raise errors.MalformedAtom(atom, - "invalid char spotted in use dep") - if not all(x.rstrip("-") for x in self.use): - raise errors.MalformedAtom( - atom, "cannot have empty use deps in use restriction") - atom = atom[0:u]+atom[u2 + 1:] - else: - sf(self, "use", None) - s = atom.find(":") - if s != -1: - i2 = atom.find(":", s + 1) - if i2 != -1: - repo_id = atom[i2 + 1:] - if not repo_id: - raise errors.MalformedAtom(atom, - "repo_id must not be empty") - elif ":" in repo_id: - raise errors.MalformedAtom(atom, - "repo_id may contain only [a-Z0-9_.-+/]") - atom = atom[:i2] - sf(self, "repo_id", repo_id) - else: - sf(self, "repo_id", None) - # slot dep. - slots = tuple(sorted(atom[s+1:].split(","))) - if not all(slots): - # if the slot char came in only due to repo_id, force slots to None - if len(slots) == 1 and i2 != -1: - slots = None - else: - raise errors.MalformedAtom(atom, - "empty slots aren't allowed") - sf(self, "slot", slots) - atom = atom[:s] - else: - sf(self, "slot", None) - sf(self, "repo_id", None) - del u, s - - sf(self, "blocks", atom[0] == "!") - if self.blocks: - atom = atom[1:] - - if atom[0] in ('<', '>'): - if atom[1] == '=': - sf(self, 'op', atom[:2]) - atom = atom[2:] - else: - sf(self, 'op', atom[0]) - atom = atom[1:] - elif atom[0] == '=': - if atom[-1] == '*': - sf(self, 'op', '=*') - atom = atom[1:-1] - else: - atom = atom[1:] - sf(self, 'op', '=') - elif atom[0] == '~': - sf(self, 'op', '~') - atom = atom[1:] - else: - sf(self, 'op', '') - sf(self, 'cpvstr', atom) - - try: - c = cpv.CPV(self.cpvstr) - except errors.InvalidCPV, e: - raise errors.MalformedAtom(orig_atom, str(e)) - sf(self, "key", c.key) - sf(self, "package", c.package) - sf(self, "category", c.category) - sf(self, "version", c.version) - sf(self, "fullver", c.fullver) - sf(self, "revision", c.revision) - - if self.op: - if self.version is None: - raise errors.MalformedAtom(orig_atom, - "operator requires a version") - elif self.version is not None: - raise errors.MalformedAtom(orig_atom, - 'versioned atom requires an operator') - sf(self, "hash", hash(orig_atom)) - sf(self, "negate_vers", negate_vers) - -def native__getattr__(self, attr): - if attr != "restrictions": - raise AttributeError(attr) - - # ordering here matters; against 24702 ebuilds for - # a non matchable atom with package as the first restriction - # 10 loops, best of 3: 206 msec per loop - # with category as the first(the obvious ordering) - # 10 loops, best of 3: 209 msec per loop - # why? because category is more likely to collide; - # at the time of this profiling, there were 151 categories. - # over 11k packages however. - r = [packages.PackageRestriction( - "package", values.StrExactMatch(self.package)), - packages.PackageRestriction( - "category", values.StrExactMatch(self.category))] - - if self.repo_id is not None: - r.insert(0, packages.PackageRestriction("repo.repo_id", - values.StrExactMatch(self.repo_id))) - - if self.fullver is not None: - if self.op == '=*': - r.append(packages.PackageRestriction( - "fullver", values.StrGlobMatch(self.fullver))) - else: - r.append(VersionMatch(self.op, self.version, self.revision, - negate=self.negate_vers)) - - if self.slot is not None: - if len(self.slot) == 1: - v = values.StrExactMatch(self.slot[0]) - else: - v = values.OrRestriction(*map(values.StrExactMatch, - self.slot)) - r.append(packages.PackageRestriction("slot", v)) - - if self.use is not None: - false_use = [x[1:] for x in self.use if x[0] == "-"] - true_use = [x for x in self.use if x[0] != "-"] - v = [] - if false_use: - v.append(values.ContainmentMatch(negate=True, - all=True, *false_use)) - - if true_use: - v.append(values.ContainmentMatch(all=True, *true_use)) - if len(v) == 1: - v = v[0] - else: - v = values.AndRestriction(*v) - r.append(packages.PackageRestriction("use", v)) - - r = tuple(r) - object.__setattr__(self, attr, r) - return r - - -native_atom_overrides = {"__init__":native_init, - "__getattr__":native__getattr__} - -try: - from pkgcore.ebuild._atom import overrides as atom_overrides -except ImportError: - atom_overrides = native_atom_overrides - - -class atom(boolean.AndRestriction): - - """Currently implements gentoo ebuild atom parsing. - - Should be converted into an agnostic dependency base. - """ - - __slots__ = ( - "blocks", "op", "negate_vers", "cpvstr", "use", - "slot", "hash", "category", "version", "revision", "fullver", - "package", "key", "repo_id") - - type = packages.package_type - negate = False - - __attr_comparison__ = ("cpvstr", "op", "blocks", "negate_vers", - "use", "slot") - - __metaclass__ = generic_equality - __inst_caching__ = True - - locals().update(atom_overrides.iteritems()) - - def __repr__(self): - if self.op == '=*': - atom = "=%s*" % self.cpvstr - else: - atom = self.op + self.cpvstr - if self.blocks: - atom = '!' + atom - attrs = [atom] - if self.use: - attrs.append('use=' + repr(self.use)) - if self.slot: - attrs.append('slot=' + repr(self.slot)) - return '<%s %s @#%x>' % ( - self.__class__.__name__, ' '.join(attrs), id(self)) - - def __reduce__(self): - return (atom, (str(self), self.negate_vers)) - - def iter_dnf_solutions(self, full_solution_expansion=False): - if full_solution_expansion: - return boolean.AndRestriction.iter_dnf_solutions( - self, full_solution_expansion=True) - return iter([[self]]) - - def cnf_solutions(self, full_solution_expansion=False): - if full_solution_expansion: - return boolean.AndRestriction.cnf_solutions( - self, full_solution_expansion=True) - return [[self]] - - def __str__(self): - if self.op == '=*': - s = "=%s*" % self.cpvstr - else: - s = self.op + self.cpvstr - if self.blocks: - s = "!" + s - if self.use: - s += "[%s]" % ",".join(self.use) - if self.slot: - s += ":%s" % ",".join(self.slot) - if self.repo_id: - s += ":%s" % self.repo_id - elif self.repo_id: - s += "::%s" % self.repo_id - return s - - def __hash__(self): - return self.hash - - def __iter__(self): - return iter(self.restrictions) - - def __getitem__(self, index): - return self.restrictions[index] - - def __cmp__(self, other): - if not isinstance(other, self.__class__): - raise TypeError("other isn't of %s type, is %s" % - (self.__class__, other.__class__)) - - c = cmp(self.category, other.category) - if c: - return c - - c = cmp(self.package, other.package) - if c: - return c - - c = cmp(self.op, other.op) - if c: - return c - - c = cpv.ver_cmp(self.version, self.revision, - other.version, other.revision) - if c: - return c - - c = cmp(self.blocks, other.blocks) - if c: - # invert it; cmp(True, False) == 1 - # want non blockers then blockers. - return -c - - c = cmp(self.negate_vers, other.negate_vers) - if c: - return c - - c = cmp(self.slot, other.slot) - if c: - return c - - return cmp(self.use, other.use) - - def intersects(self, other): - """Check if a passed in atom "intersects" this restriction's atom. - - Two atoms "intersect" if a package can be constructed that - matches both: - - if you query for just "dev-lang/python" it "intersects" both - "dev-lang/python" and ">=dev-lang/python-2.4" - - if you query for "=dev-lang/python-2.4" it "intersects" - ">=dev-lang/python-2.4" and "dev-lang/python" but not - "<dev-lang/python-2.3" - - USE and slot deps are also taken into account. - - The block/nonblock state of the atom is ignored. - """ - # Our "key" (cat/pkg) must match exactly: - if self.key != other.key: - return False - # Slot dep only matters if we both have one. If we do they - # must be identical: - if (self.slot is not None and other.slot is not None and - self.slot != other.slot): - return False - - if (self.repo_id is not None and other.repo_id is not None and - self.repo_id != other.repo_id): - return False - - # Use deps are similar: if one of us forces a flag on and the - # other forces it off we do not intersect. If only one of us - # cares about a flag it is irrelevant. - - # Skip the (very common) case of one of us not having use deps: - if self.use and other.use: - # Set of flags we do not have in common: - flags = set(self.use) ^ set(other.use) - for flag in flags: - # If this is unset and we also have the set version we fail: - if flag[0] == '-' and flag[1:] in flags: - return False - - # Remaining thing to check is version restrictions. Get the - # ones we can check without actual version comparisons out of - # the way first. - - # If one of us is unversioned we intersect: - if not self.op or not other.op: - return True - - # If we are both "unbounded" in the same direction we intersect: - if (('<' in self.op and '<' in other.op) or - ('>' in self.op and '>' in other.op)): - return True - - # Trick used here: just use the atoms as sufficiently - # package-like object to pass to these functions (all that is - # needed is a version and revision attr). - - # If one of us is an exact match we intersect if the other matches it: - if self.op == '=': - if other.op == '=*': - return self.fullver.startswith(other.fullver) - return VersionMatch( - other.op, other.version, other.revision).match(self) - if other.op == '=': - if self.op == '=*': - return other.fullver.startswith(self.fullver) - return VersionMatch( - self.op, self.version, self.revision).match(other) - - # If we are both ~ matches we match if we are identical: - if self.op == other.op == '~': - return (self.version == other.version and - self.revision == other.revision) - - # If we are both glob matches we match if one of us matches the other. - if self.op == other.op == '=*': - return (self.fullver.startswith(other.fullver) or - other.fullver.startswith(self.fullver)) - - # If one of us is a glob match and the other a ~ we match if the glob - # matches the ~ (ignoring a revision on the glob): - if self.op == '=*' and other.op == '~': - return other.fullver.startswith(self.version) - if other.op == '=*' and self.op == '~': - return self.fullver.startswith(other.version) - - # If we get here at least one of us is a <, <=, > or >=: - if self.op in ('<', '<=', '>', '>='): - ranged, other = self, other - else: - ranged, other = other, self - - if '<' in other.op or '>' in other.op: - # We are both ranged, and in the opposite "direction" (or - # we would have matched above). We intersect if we both - # match the other's endpoint (just checking one endpoint - # is not enough, it would give a false positive on <=2 vs >2) - return ( - VersionMatch( - other.op, other.version, other.revision).match(ranged) and - VersionMatch( - ranged.op, ranged.version, ranged.revision).match(other)) - - if other.op == '~': - # Other definitely matches its own version. If ranged also - # does we're done: - if VersionMatch( - ranged.op, ranged.version, ranged.revision).match(other): - return True - # The only other case where we intersect is if ranged is a - # > or >= on other's version and a nonzero revision. In - # that case other will match ranged. Be careful not to - # give a false positive for ~2 vs <2 here: - return ranged.op in ('>', '>=') and VersionMatch( - other.op, other.version, other.revision).match(ranged) - - if other.op == '=*': - # The fun one, since glob matches do not correspond to a - # single contiguous region of versions. - - # a glob match definitely matches its own version, so if - # ranged does too we're done: - if VersionMatch( - ranged.op, ranged.version, ranged.revision).match(other): - return True - if '<' in ranged.op: - # Remaining cases where this intersects: there is a - # package smaller than ranged.fullver and - # other.fullver that they both match. - - # If other.revision is not None then other does not - # match anything smaller than its own fullver: - if other.revision is not None: - return False - - # If other.revision is None then we can always - # construct a package smaller than other.fullver by - # tagging e.g. an _alpha1 on, since - # cat/pkg_beta2_alpha1_alpha1 is a valid version. - # (Yes, really. Try it if you don't believe me.) - # If and only if other also matches ranged then - # ranged will also match one of those smaller packages. - # XXX (I think, need to try harder to verify this.) - return ranged.fullver.startswith(other.version) - else: - # Remaining cases where this intersects: there is a - # package greater than ranged.fullver and - # other.fullver that they both match. - - # We can always construct a package greater than - # other.fullver by adding a digit to it. - # If and only if other also matches ranged then - # ranged will match such a larger package - # XXX (I think, need to try harder to verify this.) - return ranged.fullver.startswith(other.version) - - # Handled all possible ops. - raise NotImplementedError( - 'Someone added an op to atom without adding it to intersects') - - -def split_atom(inst): - if len(inst.restrictions) > 3: - a = packages.AndRestriction(*inst.restrictions[2:]) - elif len(inst.restrictions) == 3: - a = inst.restrictions[2] - else: - a = [] - return inst.category + "/" + inst.package, a - -def _collapsed_restrict_match(data, pkg, mode): - # mode is ignored; non applicable. - for r in data.get(pkg.key, ()): - if r.match(pkg): - return True - return False - -def generate_collapsed_restriction(atoms, negate=False): - d = {} - for a in atoms: - k = a.key - if k not in d: - d[k] = [a] - else: - d[k].append(a) - return delegate(partial(_collapsed_restrict_match, d), negate=negate) diff --git a/pkgcore/ebuild/atom_restricts.py b/pkgcore/ebuild/atom_restricts.py deleted file mode 100644 index c58fdcf..0000000 --- a/pkgcore/ebuild/atom_restricts.py +++ /dev/null @@ -1,116 +0,0 @@ -# Copyright: 2005-2006 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -# "More than one statement on a single line" -# pylint: disable-msg=C0321 - -""" -atom version restrict -""" - -from pkgcore.restrictions import packages, restriction -from pkgcore.ebuild import cpv, errors -from snakeoil.klass import generic_equality - -# TODO: change values.EqualityMatch so it supports le, lt, gt, ge, eq, -# ne ops, and convert this to it. - -class VersionMatch(restriction.base): - - """ - package restriction implementing gentoo ebuild version comparison rules - - any overriding of this class *must* maintain numerical order of - self.vals, see intersect for reason why. vals also must be a tuple. - """ - - __slots__ = ("ver", "rev", "vals", "droprev", "negate") - - __inst_caching__ = True - __metaclass__ = generic_equality - __attr_comparison__ = ('negate', 'rev', 'droprev', 'vals') - - type = packages.package_type - attr = "fullver" - - _convert_op2str = {(-1,):"<", (-1, 0): "<=", (0,):"=", - (0, 1):">=", (1,):">"} - - _convert_str2op = dict([(v, k) for k, v in _convert_op2str.iteritems()]) - del k, v - - def __init__(self, operator, ver, rev=None, negate=False, **kwd): - """ - @param operator: version comparison to do, - valid operators are ('<', '<=', '=', '>=', '>', '~') - @type operator: string - @param ver: version to base comparison on - @type ver: string - @param rev: revision to base comparison on - @type rev: None (no rev), or an int - @param negate: should the restriction results be negated; - currently forced to False - """ - - kwd["negate"] = False - super(self.__class__, self).__init__(**kwd) - sf = object.__setattr__ - sf(self, "ver", ver) - sf(self, "rev", rev) - if operator != "~" and operator not in self._convert_str2op: - raise errors.InvalidVersion(self.ver, self.rev, - "invalid operator, '%s'" % operator) - - sf(self, "negate", negate) - if operator == "~": - if ver is None: - raise ValueError( - "for ~ op, version must be something other then None") - sf(self, "droprev", True) - sf(self, "vals", (0,)) - else: - sf(self, "droprev", False) - sf(self, "vals", self._convert_str2op[operator]) - - def match(self, pkginst): - if self.droprev: - r1, r2 = None, None - else: - r1, r2 = self.rev, pkginst.revision - - return (cpv.ver_cmp(pkginst.version, r2, self.ver, r1) in self.vals) \ - != self.negate - - def __str__(self): - s = self._convert_op2str[self.vals] - - if self.negate: - n = "not " - else: - n = '' - - if self.droprev or self.rev is None: - return "ver %s%s %s" % (n, s, self.ver) - return "ver-rev %s%s %s-r%s" % (n, s, self.ver, self.rev) - - @staticmethod - def _convert_ops(inst): - if inst.negate: - if inst.droprev: - return inst.vals - return tuple(sorted(set((-1, 0, 1)).difference(inst.vals))) - return inst.vals - - def __eq__(self, other): - if self is other: - return True - if isinstance(other, self.__class__): - if self.droprev != other.droprev or self.ver != other.ver \ - or self.rev != other.rev: - return False - return self._convert_ops(self) == self._convert_ops(other) - - return False - - def __hash__(self): - return hash((self.droprev, self.ver, self.rev, self.negate, self.vals)) diff --git a/pkgcore/ebuild/conditionals.py b/pkgcore/ebuild/conditionals.py deleted file mode 100644 index a1dd74b..0000000 --- a/pkgcore/ebuild/conditionals.py +++ /dev/null @@ -1,342 +0,0 @@ -# Copyright: 2005-2007 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -"""DepSet parsing. - -Turns a DepSet (depends, rdepends, SRC_URI, license, etc) into -appropriate conditionals. -""" - -# TODO: move exceptions elsewhere, bind them to a base exception for pkgcore - -from pkgcore.restrictions import packages, values, boolean -from snakeoil.iterables import expandable_chain -from snakeoil.lists import iflatten_instance -from pkgcore.ebuild.atom import atom -from pkgcore.ebuild.errors import ParseError - -try: - from pkgcore.ebuild._depset import parse_depset -except ImportError: - parse_depset = None - -class DepSet(boolean.AndRestriction): - - """ - gentoo DepSet syntax parser - """ - - __slots__ = ("has_conditionals", "element_class", "_node_conds", - "restrictions", "_known_conditionals") - type = packages.package_type - negate = False - - __inst_caching__ = False - parse_depset = parse_depset - if parse_depset is not None: - parse_depset = staticmethod(parse_depset) - - def __init__(self, dep_str, element_class, \ - operators=None, - element_func=None): - - """ - @param dep_str: string abiding by DepSet syntax - @param operators: mapping of node -> callable for special operators - in DepSet syntax - @param element_func: if None, element_class is used for generating - elements, else it's used to generate elements. - Mainly useful for when you need to curry a few args for instance - generation, since element_class _must_ be a class - @param element_class: class of generated elements - """ - - if not isinstance(element_class, type): - # yes, this blocks non new style classes. touch cookies. - raise ValueError("element_class must be a new style class") - - sf = object.__setattr__ - sf(self, "_known_conditionals", None) - sf(self, "element_class", element_class) - if element_func is None: - element_func = element_class - - if self.parse_depset is not None: - restrictions = None - if operators is None: - has_conditionals, restrictions = self.parse_depset(dep_str, - element_func, boolean.AndRestriction, - boolean.OrRestriction) - else: - for x in operators: - if x not in ("", "||"): - break - else: - has_conditionals, restrictions = self.parse_depset(dep_str, - element_func, operators.get(""), operators.get("||")) - - if restrictions is not None: - sf(self, "_node_conds", has_conditionals) - sf(self, "restrictions", restrictions) - return - - sf(self, "restrictions", []) - if operators is None: - operators = {"||":boolean.OrRestriction, "":boolean.AndRestriction} - - raw_conditionals = [] - depsets = [self.restrictions] - - node_conds = False - words = iter(dep_str.split()) - k = None - try: - for k in words: - if ")" in k: - if ")" != k: - raise ParseError(dep_str, k) - # no elements == error. if closures don't map up, - # indexerror would be chucked from trying to pop - # the frame so that is addressed. - if not depsets[-1]: - raise ParseError(dep_str) - elif raw_conditionals[-1].endswith('?'): - node_conds = True - c = raw_conditionals[-1] - if c[0] == "!": - c = values.ContainmentMatch(c[1:-1], negate=True) - else: - c = values.ContainmentMatch(c[:-1]) - - depsets[-2].append( - packages.Conditional("use", c, tuple(depsets[-1]))) - - else: - if len(depsets[-1]) == 1: - depsets[-2].append(depsets[-1][0]) - elif raw_conditionals[-1] == '' and (len(raw_conditionals) == 1 or ('' == raw_conditionals[-2])): - # if the frame is an and and the parent is an and, collapse it in. - depsets[-2].extend(depsets[-1]) - else: - depsets[-2].append( - operators[raw_conditionals[-1]](finalize=True, - *depsets[-1])) - - raw_conditionals.pop() - depsets.pop() - - elif "(" in k: - if k != "(": - raise ParseError(dep_str, k) - - k = '' - # push another frame on - depsets.append([]) - raw_conditionals.append(k) - - elif k[-1] == '?' or k in operators: - # use conditional or custom op. - # no tokens left == bad dep_str. - k2 = words.next() - - if k2 != "(": - raise ParseError(dep_str, k2) - - # push another frame on - depsets.append([]) - raw_conditionals.append(k) - - elif "|" in k: - raise ParseError(dep_str, k) - else: - # node/element. - depsets[-1].append(element_func(k)) - - - except (RuntimeError, SystemExit, KeyboardInterrupt): - raise - except IndexError: - # [][-1] for a frame access, which means it was a parse error. - raise - except StopIteration: - if k is None: - raise - raise ParseError(dep_str, k) - except Exception, e: - raise ParseError(dep_str, e) - - # check if any closures required - if len(depsets) != 1: - raise ParseError(dep_str) - - sf(self, "_node_conds", node_conds) - sf(self, "restrictions", tuple(self.restrictions)) - - - def evaluate_depset(self, cond_dict, tristate_filter=None): - """ - @param cond_dict: container to be used for conditional collapsing, - typically is a use list - @param tristate_filter: a control; if specified, must be a container - of conditionals to lock to cond_dict. - during processing, if it's not in tristate_filter will - automatically enable the payload - (regardless of the conditionals negation) - """ - - if not self.has_conditionals: - return self - - flat_deps = self.__class__("", self.element_class) - - stack = [boolean.AndRestriction, iter(self.restrictions)] - base_restrict = [] - restricts = [base_restrict] - count = 1 - while count: - for node in stack[-1]: - if isinstance(node, self.element_class): - restricts[-1].append(node) - continue - if isinstance(node, packages.Conditional): - if not node.payload: - continue - elif tristate_filter is not None: - assert len(node.restriction.vals) == 1 - val = list(node.restriction.vals)[0] - if val in tristate_filter: - # if val is forced true, but the check is - # negation ignore it - # if !mips != mips - if (val in cond_dict) == node.restriction.negate: - continue - elif not node.restriction.match(cond_dict): - continue - if not isinstance(node.payload, tuple): - stack += [boolean.AndRestriction, iter((node.payload))] - else: - stack += [boolean.AndRestriction, iter(node.payload)] - else: - stack += [node.__class__, - iter(node.restrictions)] - count += 1 - restricts.append([]) - break - else: - stack.pop() - l = len(restricts) - if l != 1: - if restricts[-1]: - # optimization to avoid uneccessary frames. - if len(restricts[-1]) == 1: - restricts[-2].append(restricts[-1][0]) - elif stack[-1] is stack[-3] is boolean.AndRestriction: - restricts[-2].extend(restricts[-1]) - else: - restricts[-2].append(stack[-1](*restricts[-1])) - stack.pop() - count -= 1 - restricts.pop() - - object.__setattr__(flat_deps, "restrictions", tuple(base_restrict)) - return flat_deps - - @staticmethod - def find_cond_nodes(restriction_set, yield_non_conditionals=False): - conditions_stack = [] - new_set = expandable_chain(restriction_set) - for cur_node in new_set: - if isinstance(cur_node, packages.Conditional): - conditions_stack.append(cur_node.restriction) - new_set.appendleft(list(cur_node.payload) + [None]) - elif (isinstance(cur_node, boolean.base) - and not isinstance(cur_node, atom)): - new_set.appendleft(cur_node.restrictions) - elif cur_node is None: - conditions_stack.pop() - elif conditions_stack or yield_non_conditionals: # leaf - yield (cur_node, conditions_stack[:]) - - @property - def node_conds(self): - if self._node_conds is False: - object.__setattr__(self, "_node_conds", {}) - elif self._node_conds is True: - nc = {} - - always_required = set() - - for payload, restrictions in self.find_cond_nodes( - self.restrictions, True): - if not restrictions: - always_required.add(payload) - else: - if len(restrictions) == 1: - current = restrictions[0] - else: - current = values.AndRestriction(finalize=True, - *restrictions) - - nc.setdefault(payload, []).append(current) - - for k in always_required: - if k in nc: - del nc[k] - for k in nc: - nc[k] = tuple(nc[k]) - - object.__setattr__(self, "_node_conds", nc) - - return self._node_conds - - @property - def has_conditionals(self): - return bool(self._node_conds) - - @property - def known_conditionals(self): - if self._node_conds is False: - return frozenset() - if self._known_conditionals is None: - kc = set() - for payload, restrictions in self.find_cond_nodes( - self.restrictions): - kc.update(iflatten_instance(x.vals for x in restrictions)) - kc = frozenset(kc) - object.__setattr__(self, "_known_conditionals", kc) - return kc - return self._known_conditionals - - def match(self, *a): - raise NotImplementedError - - force_False = force_True = match - - def __str__(self): - return ' '.join(stringify_boolean(x) for x in self.restrictions) - - def __iter__(self): - return iter(self.restrictions) - - def __getitem__(self, key): - return self.restrictions[key] - - -def stringify_boolean(node, func=str): - """func is used to stringify the actual content. Useful for fetchables.""" - if isinstance(node, boolean.OrRestriction): - return "|| ( %s )" % " ".join(stringify_boolean(x, func) - for x in node.restrictions) - elif isinstance(node, DepSet): - return ' '.join(stringify_boolean(x, func) for x in node.restrictions) - elif isinstance(node, boolean.AndRestriction) and \ - not isinstance(node, atom): - return "( %s )" % " ".join(stringify_boolean(x, func) - for x in node.restrictions) - elif isinstance(node, packages.Conditional): - assert len(node.restriction.vals) == 1 - return "%s%s? ( %s )" % ( - node.restriction.negate and "!" or "", - list(node.restriction.vals)[0], - " ".join(stringify_boolean(x, func) for x in node.payload)) - return func(node) diff --git a/pkgcore/ebuild/const.py b/pkgcore/ebuild/const.py deleted file mode 100644 index 8815d79..0000000 --- a/pkgcore/ebuild/const.py +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright: 2005 Brian Harring <ferringb@gmail.com> -# License: GPL2 - - -""" -ebuild internal constants -""" - -eapi_capable = (0, 1) -unknown_eapi = 2 - -incrementals = ( - "USE", "FEATURES", "ACCEPT_KEYWORDS", "ACCEPT_LICENSE", - "CONFIG_PROTECT_MASK", "CONFIG_PROTECT", "PRELINK_PATH", - "PRELINK_PATH_MASK") - -metadata_keys = ( - 'DEPEND', 'RDEPEND', 'SLOT', 'SRC_URI', 'RESTRICT', 'HOMEPAGE', 'LICENSE', - 'DESCRIPTION', 'KEYWORDS', 'INHERITED', 'IUSE', 'PDEPEND', 'PROVIDE', - 'EAPI', '_mtime_', '_eclasses_') - -ACCEPT_LICENSE = () diff --git a/pkgcore/ebuild/cpv.py b/pkgcore/ebuild/cpv.py deleted file mode 100644 index e082fdd..0000000 --- a/pkgcore/ebuild/cpv.py +++ /dev/null @@ -1,314 +0,0 @@ -# Copyright: 2005 Jason Stubbs <jstubbs@gentoo.org> -# Copyright: 2005-2006 Brian Harring <ferringb@gmail.com> -# License: GPL2 - - -"""gentoo ebuild specific base package class""" - -from pkgcore.ebuild.errors import InvalidCPV - -from pkgcore.package import base -# do this to break the cycle. -from snakeoil.demandload import demandload, demand_compile_regexp -demandload(globals(), "pkgcore.ebuild:atom") - -suffix_regexp = demand_compile_regexp( - globals(), 'suffix_regexp', '^(alpha|beta|rc|pre|p)(\\d*)$') -suffix_value = {"pre": -2, "p": 1, "alpha": -4, "beta": -3, "rc": -1} - -# while the package section looks fugly, there is a reason for it- -# to prevent version chunks from showing up in the package - -valid_cat = "[a-zA-Z0-9][-a-zA-Z0-9+._]*" -parser = demand_compile_regexp( - globals(), 'parser', - "^(?P<key>(?P<category>(?:%s)(?:/%s)*)/" - "(?P<package>[a-zA-Z0-9+][a-zA-Z0-9_+]*" - "(?:-(?:[0-9]+[a-zA-Z+]{2,}[_+a-zA-Z0-9]*|[a-zA-Z+][a-zA-Z0-9+_]*))*))" - "(?:-(?P<fullver>(?P<version>(?:cvs\\.)?(?:\\d+)(?:\\.\\d+)*[a-z]?" - "(?:_(p(?:re)?|beta|alpha|rc)\\d*)*)" - "(?:-r(?P<revision>\\d+))?))?$" % - (valid_cat, valid_cat)) - - -class native_CPV(object): - - """ - base ebuild package class - - @ivar category: str category - @ivar package: str package - @ivar key: strkey (cat/pkg) - @ivar version: str version - @ivar revision: int revision - @ivar versioned_atom: atom matching this exact version - @ivar unversioned_atom: atom matching all versions of this package - @cvar _get_attr: mapping of attr:callable to generate attributes on the fly - """ - - __slots__ = ("__weakref__", "cpvstr", "key", "category", "package", - "version", "revision", "fullver") - - # if native is being used, forget trying to reuse strings. - def __init__(self, *a): - """ - Can be called with one string or with three string args. - - If called with one arg that is the cpv string. (See L{parser} - for allowed syntax). - - If called with three args they are the category, package and - version components of the cpv string respectively. - """ - l = len(a) - if l == 1: - cpvstr = a[0] - elif l == 3: - for x in a: - if not isinstance(x, basestring): - raise TypeError("all args must be strings, got %r" % (a,)) - cpvstr = "%s/%s-%s" % a - else: - raise TypeError("CPV takes 1 arg (cpvstr), or 3 (cat, pkg, ver):" - " got %r" % (a,)) - if not isinstance(cpvstr, basestring): - raise TypeError(self.cpvstr) - m = parser.match(cpvstr) - if not m: - raise InvalidCPV(cpvstr) - object.__setattr__(self, "cpvstr", cpvstr) - for k, v in m.groupdict().iteritems(): - object.__setattr__(self, k, v) - r = self.revision - if r is not None: - object.__setattr__(self, "revision", int(r)) - - def __hash__(self): - return hash(self.cpvstr) - - def __repr__(self): - return '<%s cpvstr=%s @%#8x>' % ( - self.__class__.__name__, getattr(self, 'cpvstr', None), id(self)) - - def __str__(self): - return getattr(self, 'cpvstr', 'None') - - def __cmp__(self, other): - try: - if self.cpvstr == other.cpvstr: - return 0 - - if (self.category and other.category and - self.category != other.category): - return cmp(self.category, other.category) - - if self.package and other.package and self.package != other.package: - return cmp(self.package, other.package) - - # note I chucked out valueerror, none checks on versions - # passed in. I suck, I know. - # ~harring - # fails in doing comparison of unversioned atoms against - # versioned atoms - return native_ver_cmp(self.version, self.revision, other.version, - other.revision) - except AttributeError: - return 1 - - -def native_ver_cmp(ver1, rev1, ver2, rev2): - - # If the versions are the same, comparing revisions will suffice. - if ver1 == ver2: - return cmp(rev1, rev2) - - # Split up the versions into dotted strings and lists of suffixes. - parts1 = ver1.split("_") - parts2 = ver2.split("_") - - # If the dotted strings are equal, we can skip doing a detailed comparison. - if parts1[0] != parts2[0]: - - # First split up the dotted strings into their components. - ver_parts1 = parts1[0].split(".") - ver_parts2 = parts2[0].split(".") - - # And check if CVS ebuilds come into play. If there is only - # one it wins by default. Otherwise any CVS component can - # be ignored. - if ver_parts1[0] == "cvs" and ver_parts2[0] != "cvs": - return 1 - elif ver_parts1[0] != "cvs" and ver_parts2[0] == "cvs": - return -1 - elif ver_parts1[0] == "cvs": - del ver_parts1[0] - del ver_parts2[0] - - # Pull out any letter suffix on the final components and keep - # them for later. - letters = [] - for ver_parts in (ver_parts1, ver_parts2): - if ver_parts[-1][-1].isalpha(): - letters.append(ord(ver_parts[-1][-1])) - ver_parts[-1] = ver_parts[-1][:-1] - else: - # Using -1 simplifies comparisons later - letters.append(-1) - - # OPT: Pull length calculation out of the loop - ver_parts1_len = len(ver_parts1) - ver_parts2_len = len(ver_parts2) - len_list = (ver_parts1_len, ver_parts2_len) - - # Iterate through the components - for x in xrange(max(len_list)): - - # If we've run out components, we can figure out who wins - # now. If the version that ran out of components has a - # letter suffix, it wins. Otherwise, the other version wins. - if x in len_list: - if x == ver_parts1_len: - return cmp(letters[0], 0) - else: - return cmp(0, letters[1]) - - # If the string components are equal, the numerical - # components will be equal too. - if ver_parts1[x] == ver_parts2[x]: - continue - - # If one of the components begins with a "0" then they - # are compared as floats so that 1.1 > 1.02. - if ver_parts1[x][0] == "0" or ver_parts2[x][0] == "0": - v1 = ver_parts1[x] - v2 = ver_parts2[x] - else: - v1 = int(ver_parts1[x]) - v2 = int(ver_parts2[x]) - - # If they are not equal, the higher value wins. - c = cmp(v1, v2) - if c: - return c - - # The dotted components were equal. Let's compare any single - # letter suffixes. - if letters[0] != letters[1]: - return cmp(letters[0], letters[1]) - - # The dotted components were equal, so remove them from our lists - # leaving only suffixes. - del parts1[0] - del parts2[0] - - # OPT: Pull length calculation out of the loop - parts1_len = len(parts1) - parts2_len = len(parts2) - - # Iterate through the suffixes - for x in xrange(max(parts1_len, parts2_len)): - - # If we're at the end of one of our lists, we need to use - # the next suffix from the other list to decide who wins. - if x == parts1_len: - match = suffix_regexp.match(parts2[x]) - val = suffix_value[match.group(1)] - if val: - return cmp(0, val) - return cmp(0, int("0"+match.group(2))) - if x == parts2_len: - match = suffix_regexp.match(parts1[x]) - val = suffix_value[match.group(1)] - if val: - return cmp(val, 0) - return cmp(int("0"+match.group(2)), 0) - - # If the string values are equal, no need to parse them. - # Continue on to the next. - if parts1[x] == parts2[x]: - continue - - # Match against our regular expression to make a split between - # "beta" and "1" in "beta1" - match1 = suffix_regexp.match(parts1[x]) - match2 = suffix_regexp.match(parts2[x]) - - # If our int'ified suffix names are different, use that as the basis - # for comparison. - c = cmp(suffix_value[match1.group(1)], suffix_value[match2.group(1)]) - if c: - return c - - # Otherwise use the digit as the basis for comparison. - c = cmp(int("0"+match1.group(2)), int("0"+match2.group(2))) - if c: - return c - - # Our versions had different strings but ended up being equal. - # The revision holds the final difference. - return cmp(rev1, rev2) - -fake_cat = "fake" -fake_pkg = "pkg" -def cpy_ver_cmp(ver1, rev1, ver2, rev2): - if ver1 == ver2: - return cmp(rev1, rev2) - if ver1 is None: - ver1 = '' - if ver2 is None: - ver2 = '' - c = cmp(cpy_CPV(fake_cat, fake_pkg, ver1), - cpy_CPV(fake_cat, fake_pkg, ver2)) - if c != 0: - return c - return cmp(rev1, rev2) - - -try: - # No name in module - # pylint: disable-msg=E0611 - from pkgcore.ebuild._cpv import CPV as cpy_CPV - base_CPV = cpy_CPV - ver_cmp = cpy_ver_cmp - cpy_builtin = True -except ImportError: - base_CPV = native_CPV - ver_cmp = native_ver_cmp - cpy_builtin = False - - -class CPV(base.base, base_CPV): - - """ - base ebuild package class - - @ivar category: str category - @ivar package: str package - @ivar key: strkey (cat/pkg) - @ivar version: str version - @ivar revision: int revision - @ivar versioned_atom: atom matching this exact version - @ivar unversioned_atom: atom matching all versions of this package - @cvar _get_attr: mapping of attr:callable to generate attributes on the fly - """ - - __slots__ = () - -# __metaclass__ = WeakInstMeta - -# __inst_caching__ = True - - def __repr__(self): - return '<%s cpvstr=%s @%#8x>' % ( - self.__class__.__name__, self.cpvstr, id(self)) - - @property - def versioned_atom(self): - return atom.atom("=%s" % self.cpvstr) - - @property - def unversioned_atom(self): - return atom.atom(self.key) - - def __reduce__(self): - return (self.__class__, (self.cpvstr,), None, None, None) diff --git a/pkgcore/ebuild/digest.py b/pkgcore/ebuild/digest.py deleted file mode 100644 index 22a995f..0000000 --- a/pkgcore/ebuild/digest.py +++ /dev/null @@ -1,228 +0,0 @@ -# Copyright: 2005-2006 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -""" -ebuild tree manifest/digest support -""" - -from itertools import izip -from os.path import basename, dirname, sep - -from pkgcore.chksum import errors, gpg, get_handler -from pkgcore.fs.livefs import iter_scan -from pkgcore.fs.fs import fsFile, fsDir - -from snakeoil.obj import make_SlottedDict_kls -from snakeoil.compatibility import any -from snakeoil.demandload import demandload -demandload(globals(), - "pkgcore:fetch", - "snakeoil.lists:iflatten_instance", -) - -def parse_digest(source, throw_errors=True): - d = {} - chf_keys = set(["size"]) - try: - f = None - try: - if isinstance(source, basestring): - f = open(source, "r", 32768) - else: - f = source.get_fileobj() - for line in f: - l = line.split() - if not l: - continue - if len(l) != 4: - if throw_errors: - raise errors.ParseChksumError( - source, "line count was not 4, was %i: '%s'" % ( - len(l), line)) - continue - chf = l[0].lower() - #MD5 c08f3a71a51fff523d2cfa00f14fa939 diffball-0.6.2.tar.bz2 305567 - d2 = d.get(l[2]) - if d2 is None: - d[l[2]] = {chf:long(l[1], 16), "size":long(l[3])} - else: - d2[chf] = long(l[1], 16) - chf_keys.add(chf) - except (OSError, IOError), e: - raise errors.MissingChksum(source) - except TypeError, e: - raise errors.ParseChksumError("%r" % source, e) - finally: - if f is not None and f.close: - f.close() - - kls = make_SlottedDict_kls(chf_keys) - for k, v in d.items(): - d[k] = kls(v.iteritems()) - return d - -def serialize_digest(handle, fetchables): - """ - write out a digest entry for a fetchable - - throws KeyError if needed chksums are missing. Requires at least md5 - and size chksums per fetchable. - - @param handle: file object to write to - @param fetchables: list of L{pkgcore.fetch.fetchable} instances - """ - for fetchable in iflatten_instance(fetchables, fetch.fetchable): - d = dict(fetchable.chksums) - size = d.pop("size") - try: - md5 = d.pop("md5") - handle.write("MD5 %s %s %i\n" % (get_handler('md5').long2str(md5), fetchable.filename, size)) - except KeyError: - pass - for chf, sum in d.iteritems(): - handle.write("%s %s %s %i\n" % (chf.upper(), get_handler(chf).long2str(sum), - fetchable.filename, size)) - -def serialize_manifest(pkgdir, fetchables): - """ - Write a manifest given a pkg_instance - - @param - @param - """ - handle = open(pkgdir + '/Manifest', 'w') - for file in [x for x in iter_scan(pkgdir) if isinstance(x, fsFile)]: - excludes=set(["CVS", ".svn", "Manifest"]) - if any(True for x in file.location.split(sep) if x in excludes): - continue - type = 'misc' - if 'files' in dirname(file.location): - type = 'aux' - elif basename(file.location)[-7:] == '.ebuild': - type = 'ebuild' - _write_manifest(handle, type, basename(file.location), dict(file.chksums)) - type = 'dist' - for fetchable in iflatten_instance(fetchables, fetch.fetchable): - _write_manifest(handle, type, basename(fetchable.filename), dict(fetchable.chksums)) - -def _write_manifest(handle, type, filename, chksums): - """Convenient, internal method for writing manifests""" - size = chksums.pop("size") - handle.write("%s %s %i" % (type.upper(), filename, size)) - for chf, sum in chksums.iteritems(): - handle.write(" %s %s" %(chf.upper(), get_handler(chf).long2str(sum))) - handle.write('\n') - -def convert_chksums(iterable): - for chf, sum in iterable: - chf = chf.lower() - if chf == 'size': - # explicit size entries are stupid, format has implicit size - continue - else: - yield chf, long(sum, 16) - - -def parse_manifest(source, throw_errors=True, ignore_gpg=True, - kls_override=None): - d = {} - dist, aux, ebuild, misc = {}, {}, {}, {} - types = (("DIST", dist), ("AUX", aux), ("EBUILD", ebuild), ("MISC", misc)) - files = {} - # type format (see glep 44 for exact rules) - # TYPE filename size (CHF sum)+ - # example 'type' entry, all one line - #MISC metadata.xml 219 RMD160 613195ece366b33606e71ff1753be048f2507841 SHA1 d162fb909241ef50b95a3539bdfcde95429bdf81 SHA256 cbd3a20e5c89a48a842f7132fe705bf39959f02c1025052efce8aad8a8baa8dc - # old style manifest - # CHF sum filename size - chf_types = set(["size"]) - manifest_type = 1 - try: - f = None - try: - if isinstance(source, basestring): - i = f = open(source, "r", 32768) - else: - i = f = source.get_fileobj() - if ignore_gpg: - i = gpg.skip_signatures(f) - for data in i: - line = data.split() - if not line: - continue - for t, d in types: - if line[0] != t: - continue - if len(line) % 2 != 1: - if throw_errors: - raise errors.ParseChksumError(source, - "manifest 2 entry doesn't have right " - "number of tokens, %i: %r" % - (len(line), line)) - else: - chf_types.update(line[3::2]) - # this is a trick to do pairwise collapsing; - # [size, 1] becomes [(size, 1)] - i = iter(line[3:]) - d[line[1]] = [("size", long(line[2]))] + \ - list(convert_chksums(izip(i, i))) - manifest_type = 2 - break - else: - if len(line) != 4: - if throw_errors: - raise errors.ParseChksumError(source, - "line count was not 4, was %i: %r" % - (len(line), line)) - continue - chf_types.add(line[0]) - files.setdefault(line[2], []).append( - [long(line[3]), line[0].lower(), long(line[1], 16)]) - - except (OSError, IOError, TypeError), e: - raise errors.ParseChksumError("failed parsing %r" % source, e) - finally: - if f is not None and f.close: - f.close() - - # collapse files into 4 types, convert to lower mem dicts - # doesn't handle files sublists correctly yet - for fname, data in files.iteritems(): - for t, d in types: - existing = d.get(fname) - if existing is None: - continue - break - else: - # work around portage_manifest sucking and not - # specifying all files in the manifest. - if fname.endswith(".ebuild"): - existing = ebuild.setdefault(fname, []) - else: - existing = misc.setdefault(fname, []) - - for chksum in data: - if existing: - if existing[0][1] != chksum[0]: - if throw_errors: - raise errors.ParseChksumError(source, - "size collision for file %s" % fname) - else: - existing.append(chksum[1:]) - else: - existing.append(("size", chksum[0])) - existing.append(chksum[1:]) - - del files - - # finally convert it to slotted dict for memory savings. - kls = make_SlottedDict_kls(x.lower() for x in chf_types) - ret = [] - for t, d in types: - if kls_override is None: - for k, v in d.items(): - d[k] = kls(v) - else: - d = kls_override((k, kls(v)) for k, v in d.iteritems()) - ret.append(d) - return ret, manifest_type diff --git a/pkgcore/ebuild/domain.py b/pkgcore/ebuild/domain.py deleted file mode 100644 index 7747bfd..0000000 --- a/pkgcore/ebuild/domain.py +++ /dev/null @@ -1,436 +0,0 @@ -# Copyright: 2005-2007 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -""" -gentoo configuration domain -""" - -# XXX doc this up better... - -from itertools import izip -from os.path import isfile - -import pkgcore.config.domain -from pkgcore.config import ConfigHint -from pkgcore.restrictions.delegated import delegate -from pkgcore.restrictions import packages, values -from pkgcore.ebuild.atom import generate_collapsed_restriction -from pkgcore.repository import multiplex, visibility -from pkgcore.interfaces.data_source import local_source -from pkgcore.config.errors import BaseException -from pkgcore.ebuild import const -from pkgcore.ebuild.profiles import incremental_expansion -from pkgcore.ebuild.misc import (collapsed_restrict_to_data, - non_incremental_collapsed_restrict_to_data) -from pkgcore.util.parserestrict import parse_match - -from snakeoil.lists import stable_unique, unstable_unique -from snakeoil.compatibility import any -from snakeoil.mappings import ProtectedDict -from snakeoil.fileutils import iter_read_bash -from snakeoil.currying import partial -from snakeoil.demandload import demandload -demandload(globals(), - 'errno', - 'pkgcore.fs.livefs:iter_scan', - 'pkgcore.fs.fs:fsFile', -) - -class MissingFile(BaseException): - def __init__(self, filename, setting): - BaseException.__init__(self, - "setting %s points at %s, which doesn't exist." - % (setting, filename)) - self.file, self.setting = filename, setting - -class Failure(BaseException): - def __init__(self, text): - BaseException.__init__(self, "domain failure: %s" % (text,)) - self.text = text - - -def package_keywords_splitter(val): - v = val.split() - return parse_match(v[0]), stable_unique(v[1:]) - - -# ow ow ow ow ow ow.... -# this manages a *lot* of crap. so... this is fun. -# -# note also, that this is rather ebuild centric. it shouldn't be, and -# should be redesigned to be a seperation of configuration -# instantiation manglers, and then the ebuild specific chunk (which is -# selected by config) -# ~harring - - -def generate_masking_restrict(masks): - # if it's masked, it's not a match - return generate_collapsed_restriction(masks, negate=True) - -def generate_unmasking_restrict(unmasks): - return generate_collapsed_restriction(unmasks) - - -class domain(pkgcore.config.domain.domain): - - # XXX ouch, verify this crap and add defaults and stuff - _types = { - 'profile': 'ref:profile', 'fetcher': 'ref:fetcher', - 'repositories': 'lazy_refs:repo', 'vdb': 'lazy_refs:repo', - 'name': 'str', - } - for _thing in list(const.incrementals) + ['bashrc']: - _types[_thing] = 'list' - for _thing in [ - 'package.mask', 'package.keywords', 'package.license', 'package.use', - 'package.unmask']: - _types[_thing] = 'list' - for _thing in ['root', 'CHOST', 'CFLAGS', 'PATH', 'PORTAGE_TMPDIR', - 'DISTCC_PATH', 'DISTCC_DIR', 'CCACHE_DIR']: - _types[_thing] = 'str' - - # TODO this is missing defaults - pkgcore_config_type = ConfigHint( - _types, typename='domain', - required=['repositories', 'profile', 'vdb', 'fetcher', 'name'], - allow_unknowns=True) - - del _types, _thing - - def __init__(self, profile, repositories, vdb, name=None, - root='/', incrementals=const.incrementals, **settings): - # voodoo, unfortunately (so it goes) - # break this up into chunks once it's stabilized (most of code - # here has already, but still more to add) - settings.setdefault('ACCEPT_LICENSE', const.ACCEPT_LICENSE) - - # map out sectionname -> config manager immediately. - repositories_collapsed = [r.collapse() for r in repositories] - repositories = [r.instantiate() for r in repositories_collapsed] - vdb_collapsed = [r.collapse() for r in vdb] - vdb = [r.instantiate() for r in vdb_collapsed] - self.named_repos = dict( - (collapsed.name, repo) for (collapsed, repo) in izip( - repositories_collapsed, repositories)) - self.named_repos.update( - (collapsed.name, repo) for (collapsed, repo) in izip( - vdb_collapsed, vdb)) - self.named_repos.pop(None, None) - - pkg_maskers = set(profile.masks) - for r in repositories: - pkg_maskers.update(r.default_visibility_limiters) - pkg_maskers = list(pkg_maskers) - pkg_unmaskers, pkg_keywords, pkg_license = [], [], [] - pkg_use = [] - - for key, val, action in ( - ("package.mask", pkg_maskers, parse_match), - ("package.unmask", pkg_unmaskers, parse_match), - ("package.keywords", pkg_keywords, package_keywords_splitter), - ("package.license", pkg_license, package_keywords_splitter), - ("package.use", pkg_use, package_keywords_splitter)): - - for fp in settings.pop(key, ()): - try: - if isfile(fp): - val.extend(action(x) for x in iter_read_bash(fp)) - else: - # Ok, so it might not be a dir, but iter_scan'ing it - # means we get a nice exception w/o having to set it - # ourselves. - for file in iter_scan(fp): - if (not isinstance(file, fsFile) or - any(True for thing in file.location.split('/') - if thing.startswith('.'))): - continue - val.extend(action(x) for x in iter_read_bash(file.location)) - except (IOError, OSError), e: - if e.errno == errno.ENOENT: - raise MissingFile(settings[key], key) - raise Failure("failed reading '%s': %s" % (fp, e)) - except ValueError, e: - raise Failure("failed reading '%s': %s" % (fp, e)) - - self.name = name - settings.setdefault("PKGCORE_DOMAIN", name) - for x in incrementals: - if isinstance(settings.get(x), basestring): - settings[x] = set(settings[x].split()) - - for x, v in profile.default_env.iteritems(): - if x in settings: - if x in incrementals: - if isinstance(v, basestring): - v = set(v.split()) - else: - v = set(v) - incremental_expansion(v, settings[x]) - settings[x] = v - else: - if x in incrementals: - if isinstance(v, basestring): - v = set(v.split()) - settings[x] = v - else: - settings[x] = v - - # use is collapsed; now stack use_expand. - use = settings.setdefault("USE", set()) - - # hackish implementation; if test is on, flip on the flag - if "test" in settings.get("FEATURES", ()): - use.add("test") - - self.use_expand = frozenset(profile.use_expand) - self.use_expand_hidden = frozenset(profile.use_expand_hidden) - for u in profile.use_expand: - v = settings.get(u) - if v is None: - continue - u2 = u.lower()+"_" - use.update(u2 + x for x in settings[u].split()) - - # visibility mask... - # if ((package.mask or visibility) and not package.unmask) - # or not (package.keywords or accept_keywords) - - vfilter = packages.AndRestriction(finalize=False, - disable_inst_caching=False) - r = None - if pkg_maskers: - r = generate_masking_restrict(pkg_maskers) - if pkg_unmaskers: - if r is None: - # unmasking without masking... 'k (wtf?) - r = generate_unmasking_restrict(pkg_unmaskers) - else: - r = packages.OrRestriction( - r, generate_unmasking_restrict(pkg_unmaskers), - disable_inst_caching=True) - if r: - vfilter.add_restriction(r) - del pkg_unmaskers, pkg_maskers - - license, default_keywords = [], [] - master_license = [] - for k, v in (("ACCEPT_KEYWORDS", default_keywords), - ("ACCEPT_LICENSE", master_license)): - if k not in settings: - raise Failure("No %s setting detected from profile, " - "or user config" % k) - s = set() - incremental_expansion(s, settings[k], "while expanding %s: " % k) - v.extend(s) - settings[k] = v - - - self.use = use - - if "ARCH" not in settings: - raise Failure( - "No ARCH setting detected from profile, or user config") - - self.arch = settings["ARCH"] - - # ~amd64 -> [amd64, ~amd64] - for x in default_keywords[:]: - if x.startswith("~"): - default_keywords.append(x.lstrip("~")) - default_keywords = unstable_unique(default_keywords + [self.arch]) - - vfilter.add_restriction(self.make_keywords_filter( - self.arch, default_keywords, pkg_keywords, - incremental="package.keywords" in incrementals)) - - del default_keywords - # we can finally close that fricking - # "DISALLOW NON FOSS LICENSES" bug via this >:) - if master_license: - vfilter.add_restriction(self.make_license_filter( - master_license, license)) - - del master_license, license - - # if it's made it this far... - - self.root = settings["ROOT"] = root - self.settings = settings - - bashrc = list(profile.bashrc) - - if "bashrc" in self.settings: - for data in self.settings['bashrc']: - source = local_source(data) - # this is currently local-only so a get_path check is ok - # TODO make this more general - if source.get_path() is None: - raise Failure( - 'user-specified bashrc %r does not exist' % (data,)) - bashrc.append(source) - - # stack use stuff first, then profile. - # could do an intersect up front to pull out the forced disabled - # also, although that code would be fugly - self.enabled_use = collapsed_restrict_to_data( - ((packages.AlwaysTrue, self.use), - (packages.AlwaysTrue, [self.arch])), - profile.pkg_use.iteritems(), - pkg_use) - self.forced_use = collapsed_restrict_to_data( - profile.forced_use.iteritems(), - ((packages.AlwaysTrue, [self.arch]),)) - self.disabled_use = collapsed_restrict_to_data( - profile.masked_use.iteritems()) - - self.settings["bashrc"] = bashrc - self.repos = [] - self.vdb = [] - self.configured_named_repos = {} - self.filtered_named_repos = {} - - rev_names = dict((repo, name) for name, repo in self.named_repos.iteritems()) - - - for l, repos, filtered in ((self.repos, repositories, True), - (self.vdb, vdb, False)): - - for repo in repos: - if not repo.configured: - pargs = [repo] - try: - for x in repo.configurables: - if x == "domain": - pargs.append(self) - elif x == "settings": - pargs.append(ProtectedDict(settings)) - elif x == "profile": - pargs.append(profile) - else: - pargs.append(getattr(self, x)) - except AttributeError, ae: - raise Failure("failed configuring repo '%s': " - "configurable missing: %s" % (repo, ae)) - wrapped_repo = repo.configure(*pargs) - else: - wrapped_repo = repo - key = rev_names.get(repo) - self.configured_named_repos[key] = wrapped_repo - if filtered: - wrapped_repo = visibility.filterTree(wrapped_repo, - vfilter, True) - self.filtered_named_repos[key] = wrapped_repo - l.append(wrapped_repo) - - if profile.virtuals: - l = [x for x in (getattr(v, 'old_style_virtuals', None) - for v in self.vdb) if x is not None] - profile_repo = profile.make_virtuals_repo( - multiplex.tree(*repositories), *l) - self.named_repos["profile virtuals"] = profile_repo - self.filtered_named_repos["profile virtuals"] = profile_repo - self.configured_named_repos["profile virtuals"] = profile_repo - self.repos = [profile_repo] + self.repos - - def make_license_filter(self, master_license, pkg_licenses): - data = collapsed_restrict_to_data( - ((packages.AlwaysTrue, master_license),), - pkg_licenses) - return delegate(partial(self.apply_license_filter, data)) - - def apply_license_filter(self, data, pkg, mode): - # note we're not honoring mode; it's always match. - # reason is that of not turning on use flags to get acceptible license - # pairs. - # maybe change this down the line? - allowed_licenses = data.pull_data(pkg) - for and_pair in pkg.license.dnf_solutions(): - for license in and_pair: - if license not in allowed_licenses: - break - else: - # tiz fine. - return True - return False - - def make_keywords_filter(self, arch, default_keys, pkg_keywords, - incremental=False): - """Generates a restrict that matches iff the keywords are allowed.""" - if not pkg_keywords: - return packages.PackageRestriction( - "keywords", values.ContainmentMatch(*default_keys)) - - if "~" + arch.lstrip("~") not in default_keys: - # stable; thus empty entries == ~arch - unstable = "~" + arch - def f(r, v): - if not v: - return r, unstable - return r, v - data = collapsed_restrict_to_data( - ((packages.AlwaysTrue, default_keys),), - (f(*i) for i in pkg_keywords)) - else: - if incremental: - f = collapsed_restrict_to_data - else: - f = non_incremental_collapsed_restrict_to_data - data = f(((packages.AlwaysTrue, default_keys),), - pkg_keywords) - - if incremental: - raise NotImplementedError(self.incremental_apply_keywords_filter) - #f = self.incremental_apply_keywords_filter - else: - f = self.apply_keywords_filter - return delegate(partial(f, data)) - - @staticmethod - def incremental_apply_keywords_filter(data, pkg, mode): - # note we ignore mode; keywords aren't influenced by conditionals. - # note also, we're not using a restriction here. this is faster. - allowed = data.pull_data(pkg) - return any(True for x in pkg.keywords if x in allowed) - - @staticmethod - def apply_keywords_filter(data, pkg, mode): - # note we ignore mode; keywords aren't influenced by conditionals. - # note also, we're not using a restriction here. this is faster. - allowed = data.pull_data(pkg) - if '**' in allowed: - return True - if "*" in allowed: - for k in pkg.keywords: - if k[0] not in "-~": - return True - if "~*" in allowed: - for k in pkg.keywords: - if k[0] == "~": - return True - return any(True for x in pkg.keywords if x in allowed) - - def make_per_package_use(self, default_use, pkg_use): - if not pkg_use: - return default_use, ((), {}) - return collapsed_restrict_to_data(default_use, pkg_use) - - def get_package_use(self, pkg): - disabled = self.disabled_use.pull_data(pkg) - enabled = self.enabled_use.pull_data(pkg) - immutable = self.forced_use.pull_data(pkg, False) - if disabled: - if enabled is self.enabled_use.defaults: - enabled = set(enabled) - if immutable is self.forced_use.defaults: - immutable = set(immutable) - elif immutable: - if enabled is self.enabled_use.defaults: - enabled = set(enabled) - else: - return immutable, enabled - enabled.update(immutable) - enabled.difference_update(disabled) - immutable.update(disabled) - - return immutable, enabled diff --git a/pkgcore/ebuild/ebd.py b/pkgcore/ebuild/ebd.py deleted file mode 100644 index df8519f..0000000 --- a/pkgcore/ebuild/ebd.py +++ /dev/null @@ -1,666 +0,0 @@ -# Copyright: 2005-2007 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -""" -EBuild Daemon (ebd), main high level interface to ebuild execution env -(ebuild.sh being part of it). - -Wraps L{pkgcore.ebuild.processor} functionality into a higher level -api, per phase methods for example -""" - - -import os, errno, shutil - -from pkgcore.interfaces import format, data_source -from pkgcore.ebuild.processor import \ - request_ebuild_processor, release_ebuild_processor, \ - expected_ebuild_env, chuck_UnhandledCommand -from pkgcore.os_data import portage_gid, portage_uid -from pkgcore.spawn import ( - spawn_bash, spawn, is_sandbox_capable, is_fakeroot_capable) -from pkgcore.os_data import xargs -from pkgcore.ebuild.const import eapi_capable -from pkgcore.interfaces import observer -from pkgcore.ebuild.ebuild_built import fake_package_factory, package -from snakeoil.currying import post_curry, pretty_docs -from snakeoil.osutils import ensure_dirs, normpath, join as pjoin - -from snakeoil.demandload import demandload -demandload(globals(), - "pkgcore.log:logger", - "pkgcore.package.mutated:MutatedPkg", -) - - -def _reset_env_data_source(method): - return method - - # unreachable code. --charlie - #def store_env_data_wrapper(self, *args, **kwds): - # try: - # return method(self, *args, **kwds) - # finally: - # # note that we're *not* return'ing anything ourselves. - # # we want the original val to slide back - # if self.env_data_source is None: - # try: - # fp = self.env["PORT_ENV_FILE"] - # f = self.env_data.get_fileobj() - # f.seek(0, 0) - # f.truncate(0) - # f.write(open(fp, "r").read()) - # del f, fp - # except (IOError, OSError), oe: - # if oe.errno != errno.ENOENT: - # raise - - #store_env_data_wrapper.__doc__ = method.__doc__ - #return store_env_data_wrapper - - -class ebd(object): - - def __init__(self, pkg, initial_env=None, env_data_source=None, - features=None, observer=None, clean=False, tmp_offset=None): - """ - @param pkg: - L{ebuild package instance<pkgcore.ebuild.ebuild_src.package>} - instance this env is being setup for - @param initial_env: initial environment to use for this ebuild - @param env_data_source: a L{pkgcore.interfaces.data_source} instance - to restore the environment from- used for restoring the - state of an ebuild processing, whether for unmerging, or - walking phases during building - @param features: ebuild features, hold over from portage, - will be broken down at some point - """ - - if not hasattr(self, "observer"): - self.observer = observer - if pkg.eapi not in eapi_capable: - raise TypeError( - "pkg isn't of a supported eapi!, %i not in %s for %s" % ( - pkg.eapi, eapi_capable, pkg)) - - if initial_env is not None: - # copy. - self.env = dict(initial_env) - for x in ("USE", "ACCEPT_LICENSE"): - if x in self.env: - del self.env[x] - else: - self.env = {} - - # temp hack. - if "PYTHONPATH" in os.environ: - self.env["PYTHONPATH"] = os.environ["PYTHONPATH"] - if "PKGCORE_DEBUG" in os.environ: - self.env["PKGCORE_DEBUG"] = str(int(os.environ["PKGCORE_DEBUG"])) - - self.env.setdefault("ROOT", "/") - self.env_data_source = env_data_source - if env_data_source is not None and \ - not isinstance(env_data_source, data_source.base): - raise TypeError( - "env_data_source must be None, or a pkgcore.data_source.base " - "derivative: %s: %s" % ( - env_data_source.__class__, env_data_source)) - - if features is None: - features = self.env.get("FEATURES", ()) - - self.features = set(x.lower() for x in features) - - self.env["FEATURES"] = ' '.join(sorted(self.features)) - - expected_ebuild_env(pkg, self.env, env_source_override=self.env_data_source) - - self.env["USE"] = ' '.join(str(x) for x in pkg.use) - self.env["INHERITED"] = ' '.join(pkg.data.get("_eclasses_", ())) - self.env["SLOT"] = pkg.slot - self.env["FINALIZED_RESTRICT"] = ' '.join(str(x) for x in pkg.restrict) - - self.restrict = pkg.restrict - - for x in ("sandbox", "userpriv", "fakeroot"): - setattr(self, x, self.feat_or_bool(x) and not (x in self.restrict)) - if self.fakeroot: - logger.warn("disabling fakeroot; unusable till coreutils/fakeroot" + - " interaction is fixed") - self.fakeroot = False - if self.userpriv and os.getuid() != 0: - self.userpriv = False - - if "PORT_LOGDIR" in self.env: - self.logging = pjoin(self.env["PORT_LOGDIR"], - pkg.cpvstr+".log") - del self.env["PORT_LOGDIR"] - else: - self.logging = False - - self.env["XARGS"] = xargs - - self.bashrc = self.env.get("bashrc", ()) - if self.bashrc: - del self.env["bashrc"] - - self.pkg = pkg - self.eapi = pkg.eapi - wipes = [k for k, v in self.env.iteritems() - if not isinstance(v, basestring)] - for k in wipes: - del self.env[k] - del wipes, k, v - - self.set_op_vars(tmp_offset) - self.clean_at_start = clean - self.clean_needed = False - - def start(self): - if self.clean_at_start: - self.clean_needed = True - if not self.cleanup(): - return False - self.setup_workdir() - self.setup_env_data_source() - self.clean_needed = True - return True - - def set_op_vars(self, tmp_offset): - # don't fool with this, without fooling with setup. - self.base_tmpdir = self.env["PORTAGE_TMPDIR"] - self.tmpdir = normpath(pjoin(self.base_tmpdir, "portage")) - if tmp_offset: - self.tmpdir = pjoin(self.tmpdir, - tmp_offset.strip(os.path.sep)) - - self.builddir = pjoin(self.tmpdir, self.env["CATEGORY"], - self.env["PF"]) - for x, y in (("T", "temp"), ("WORKDIR", "work"), ("D", "image"), - ("HOME", "homedir")): - self.env[x] = pjoin(self.builddir, y) +"/" - - self.env["IMAGE"] = self.env["D"] - - def get_env_source(self): - return data_source.data_source( - open(pjoin(self.env["T"], "environment"), "r").read()) - - def setup_env_data_source(self): - if not ensure_dirs(self.env["T"], mode=0770, gid=portage_gid, - minimal=True): - raise format.FailedDirectory(self.env['T'], - "%s doesn't fulfill minimum mode %o and gid %i" % ( - self.env['T'], 0770, portage_gid)) - - if self.env_data_source is not None: - fp = pjoin(self.env["T"], "environment") - # load data first (might be a local_source), *then* right - # if it's a src_ebuild being installed, trying to do two steps - # stomps the local_sources data. - data = self.env_data_source.get_fileobj().read() - open(fp, "w").write(data) - del data - - def setup_logging(self): - if self.logging and not ensure_dirs(os.path.dirname(self.logging), - mode=02770, gid=portage_gid): - raise format.FailedDirectory( - os.path.dirname(self.logging), - "PORT_LOGDIR, desired mode 02770 and gid %i" % portage_gid) - - def setup_workdir(self): - # ensure dirs. - for k in ("HOME", "T", "WORKDIR", "D"): - if not ensure_dirs(self.env[k], mode=04770, - gid=portage_gid, minimal=True): - raise format.FailedDirectory( - self.env[k], - "%s doesn't fulfill minimum mode %o and gid %i" % ( - k, 0770, portage_gid)) - # XXX hack, just 'til pkgcore controls these directories - if (os.stat(self.env[k]).st_mode & 02000): - logger.warn("%s ( %s ) is setgid" % (self.env[k], k)) - - - @_reset_env_data_source - def _generic_phase(self, phase, userpriv, sandbox, fakeroot, - extra_handlers=None): - """ - @param phase: phase to execute - @param userpriv: will we drop to - L{portage_uid<pkgcore.os_data.portage_uid>} and - L{portage_gid<pkgcore.os_data.portage_gid>} access for this phase? - @param sandbox: should this phase be sandboxed? - @param fakeroot: should the phase be fakeroot'd? Only really useful - for install phase, and is mutually exclusive with sandbox - """ - ebd = request_ebuild_processor(userpriv=(self.userpriv and userpriv), - sandbox=(self.sandbox and sandbox and is_sandbox_capable()), - fakeroot=(self.fakeroot and fakeroot and is_fakeroot_capable())) - try: - ebd.prep_phase(phase, self.env, sandbox=self.sandbox, - logging=self.logging) - ebd.write("start_processing") - if not ebd.generic_handler(additional_commands=extra_handlers): - raise format.GenericBuildError( - phase + ": Failed building (False/0 return from handler)") - - except Exception, e: - ebd.shutdown_processor() - release_ebuild_processor(ebd) - if isinstance(e, (SystemExit, format.GenericBuildError)): - raise - raise format.GenericBuildError( - phase + ": Caught exception while building: %s" % e) - - release_ebuild_processor(ebd) - return True - - def cleanup(self, disable_observer=False): - if not self.clean_needed or not os.path.exists(self.builddir): - return True - if disable_observer: - return self.do_cleanup(disable_observer=disable_observer) - return self.do_cleanup() - - @observer.decorate_build_method("cleanup") - def do_cleanup(self): - try: - shutil.rmtree(self.builddir) - # try to wipe the cat dir; if not empty, ignore it - try: - os.rmdir(os.path.dirname(self.builddir)) - except OSError, e: - if e.errno != errno.ENOTEMPTY: - raise - except OSError, oe: - raise format.GenericBuildError( - "clean: Caught exception while cleansing: %s" % oe) - return True - - def feat_or_bool(self, name, extra_env=None): - if name in self.env: - v = bool(self.env[name]) - del self.env[name] - name = name.lower() - if v: - self.features.add(name) - else: - if name in self.features: - self.features.remove(name) - elif extra_env is not None and name in extra_env: - v = bool(extra_env[name]) - if v: - self.features.add(name.lower()) - else: - self.features.remove(name.lower()) - else: - v = name.lower() in self.features - return v - - -class setup_mixin(object): - - setup_is_for_src = True - - def setup(self): - self.setup_logging() - - additional_commands = {} - phase_name = "setup-binpkg" - if self.setup_is_for_src: - phase_name = "setup" - - ebdp = request_ebuild_processor(userpriv=False, sandbox=False) - if self.setup_is_for_src: - additional_commands["request_inherit"] = \ - post_curry(ebdp.__class__._inherit, self.eclass_cache) - additional_commands["request_profiles"] = self._request_bashrcs - - try: - ebdp.prep_phase(phase_name, self.env, sandbox=self.sandbox, - logging=self.logging) - ebdp.write("start_processing") - if not ebdp.generic_handler( - additional_commands=additional_commands): - raise format.GenericBuildError( - "setup: Failed building (False/0 return from handler)") - - except Exception, e: - # regardless of what occured, we kill the processor. - ebdp.shutdown_processor() - release_ebuild_processor(ebdp) - # either we know what it is, or it's a shutdown. re-raise - if isinstance(e, (SystemExit, format.GenericBuildError)): - raise - # wrap. - raise format.GenericBuildError( - "setup: Caught exception while building: " + str(e)) - - release_ebuild_processor(ebdp) - return True - - def _request_bashrcs(self, ebd, a): - if a is not None: - chuck_UnhandledCommand(ebd, "bashrc request with arg"+str(a)) - for source in self.bashrc: - if source.get_path is not None: - ebd.write("path\n%s" % source.get_path()) - elif source.get_data is not None: - raise NotImplementedError - else: - chuck_UnhandledCommand( - ebd, "bashrc request: unable to process bashrc " - "due to source '%s' due to lacking usable get_*" % ( - source,)) - if not ebd.expect("next"): - chuck_UnhandledCommand( - ebd, "bashrc transfer, didn't receive 'next' response. " - "failure?") - ebd.write("end_request") - - -class install_op(ebd, format.install): - """ - phase operations and steps for install execution - """ - - preinst = pretty_docs( - observer.decorate_build_method("preinst")( - post_curry( - ebd._generic_phase, "preinst", False, False, False)), - "run the postinst phase") - postinst = pretty_docs( - observer.decorate_build_method("postinst")( - post_curry( - ebd._generic_phase, "postinst", False, False, False)), - "run the postinst phase") - - -class uninstall_op(ebd, format.uninstall): - """ - phase operations and steps for uninstall execution - """ - - def __init__(self, *args, **kwargs): - kwargs["tmp_offset"] = "unmerge" - ebd.__init__(self, *args, **kwargs) - - prerm = pretty_docs( - observer.decorate_build_method("prerm")( - post_curry( - ebd._generic_phase, "prerm", False, False, False)), - "run the prerm phase") - postrm = pretty_docs( - observer.decorate_build_method("postrm")( - post_curry( - ebd._generic_phase, "postrm", False, False, False)), - "run the postrm phase") - - -class replace_op(format.replace, install_op, uninstall_op): - def __init__(self, *args, **kwargs): - ebd.__init__(self, *args, **kwargs) - - -class buildable(ebd, setup_mixin, format.build): - - """ - build operation - """ - - _built_class = package - - # XXX this is unclean- should be handing in strictly what is build - # env, rather then dumping domain settings as env. - def __init__(self, pkg, domain_settings, eclass_cache, fetcher, - observer=None, **kwargs): - - """ - @param pkg: L{pkgcore.ebuild.ebuild_src.package} instance we'll be - building - @param domain_settings: dict bled down from the domain configuration; - basically initial env - @param eclass_cache: the L{eclass_cache<pkgcore.ebuild.eclass_cache>} - we'll be using - @param fetcher: a L{pkgcore.fetch.base.fetcher} instance to use to - access our required files for building - """ - - format.build.__init__(self, observer=observer) - ebd.__init__(self, pkg, initial_env=domain_settings, - features=domain_settings["FEATURES"], **kwargs) - - self.env["FILESDIR"] = pjoin(os.path.dirname(pkg.ebuild.get_path()), "files") - self.eclass_cache = eclass_cache - self.env["ECLASSDIR"] = eclass_cache.eclassdir - self.env["PORTDIR"] = eclass_cache.portdir - - self.fetcher = fetcher - - self.run_test = self.feat_or_bool("test", domain_settings) - if "test" in self.restrict: - self.run_test = False - elif "test" not in pkg.use: - if self.run_test: - logger.warn("disabling test for %s due to test use flag being disabled") - self.run_test = False - - # XXX minor hack - path = self.env["PATH"].split(":") - - for s, default in (("DISTCC", ".distcc"), ("CCACHE", "ccache")): - b = (self.feat_or_bool(s, domain_settings) - and not s in self.restrict) - setattr(self, s.lower(), b) - if b: - # looks weird I realize, but - # pjoin("/foor/bar", "/barr/foo") == "/barr/foo" - # and pjoin("/foo/bar",".asdf") == "/foo/bar/.asdf" - self.env.setdefault(s+"_DIR", pjoin(self.tmpdir, default)) - path.insert(0, "/usr/lib/%s/bin" % s.lower()) - else: - for y in ("_PATH", "_DIR"): - if s+y in self.env: - del self.env[s+y] - path = [piece for piece in path if piece] - self.env["PATH"] = ":".join(path) - self.fetchables = pkg.fetchables[:] - self.env["A"] = ' '.join(set(x.filename - for x in self.fetchables)) - - if self.setup_is_for_src: - self.init_distfiles_env() - - def init_distfiles_env(self): - # cvs/svn ebuilds need to die. - distdir_write = self.fetcher.get_storage_path() - if distdir_write is None: - raise format.GenericBuildError("no usable distdir was found " - "for PORTAGE_ACTUAL_DISTDIR from fetcher %s" % self.fetcher) - self.env["PORTAGE_ACTUAL_DISTDIR"] = distdir_write - self.env["DISTDIR"] = normpath( - pjoin(self.builddir, "distdir")) - for x in ("PORTAGE_ACTUAL_DISTDIR", "DISTDIR"): - self.env[x] = os.path.realpath(self.env[x]).rstrip("/") + "/" - - def setup_distfiles(self): - # added to protect against no-auto usage in pebuild. - if not hasattr(self, 'files'): - self.fetch() - - if self.files: - try: - if os.path.exists(self.env["DISTDIR"]): - if (os.path.isdir(self.env["DISTDIR"]) - and not os.path.islink(self.env["DISTDIR"])): - shutil.rmtree(self.env["DISTDIR"]) - else: - os.unlink(self.env["DISTDIR"]) - - except OSError, oe: - raise format.FailedDirectory( - self.env["DISTDIR"], - "failed removing existing file/dir/link at: exception %s" - % oe) - - if not ensure_dirs(self.env["DISTDIR"], mode=0770, - gid=portage_gid): - raise format.FailedDirectory( - self.env["DISTDIR"], - "failed creating distdir symlink directory") - - try: - for src, dest in [ - (k, pjoin(self.env["DISTDIR"], v.filename)) - for (k, v) in self.files.items()]: - os.symlink(src, dest) - - except OSError, oe: - raise format.GenericBuildError( - "Failed symlinking in distfiles for src %s -> %s: %s" % ( - src, dest, str(oe))) - - @observer.decorate_build_method("setup") - def setup(self): - """ - execute the setup phase, mapping out to pkg_setup in the ebuild - - necessarily dirs are created as required, and build env is - initialized at this point - """ - if self.distcc: - for p in ("", "/lock", "/state"): - if not ensure_dirs(pjoin(self.env["DISTCC_DIR"], p), - mode=02775, gid=portage_gid): - raise format.FailedDirectory( - pjoin(self.env["DISTCC_DIR"], p), - "failed creating needed distcc directory") - if self.ccache: - # yuck. - st = None - try: - st = os.stat(self.env["CCACHE_DIR"]) - except OSError: - st = None - if not ensure_dirs(self.env["CCACHE_DIR"], mode=02775, - gid=portage_gid): - raise format.FailedDirectory( - self.env["CCACHE_DIR"], - "failed creation of ccache dir") - - # XXX this is more then mildly stupid. - st = os.stat(self.env["CCACHE_DIR"]) - try: - if st.st_gid != portage_gid or (st.st_mode & 02775) != 02775: - try: - cwd = os.getcwd() - except OSError: - cwd = "/" - try: - # crap. - os.chmod(self.env["CCACHE_DIR"], 02775) - os.chown(self.env["CCACHE_DIR"], -1, portage_gid) - os.chdir(cwd) - if 0 != spawn(["chgrp", "-R", str(portage_gid), - self.env["CCACHE_DIR"]]): - raise format.FailedDirectory( - self.env["CCACHE_DIR"], - "failed changing ownership for CCACHE_DIR") - if 0 != spawn_bash( - "find '%s' -type d -print0 | %s --null chmod 02775" - % (self.env["CCACHE_DIR"], xargs)): - raise format.FailedDirectory( - self.env["CCACHE_DIR"], - "failed correcting perms for CCACHE_DIR") - - if 0 != spawn_bash( - "find '%s' -type f -print0 | %s --null chmod 0775" - % (self.env["CCACHE_DIR"], xargs)): - raise format.FailedDirectory( - self.env["CCACHE_DIR"], - "failed correcting perms for CCACHE_DIR") - finally: - os.chdir(cwd) - except OSError: - raise format.FailedDirectory( - self.env["CCACHE_DIR"], - "failed ensuring perms/group owner for CCACHE_DIR") - return setup_mixin.setup(self) - - def configure(self): - """ - execute the configure phase. - - does nothing if the pkg is EAPI=0 (that spec lacks a seperated - configure phase). - """ - if self.eapi > 0: - return self._generic_phase("configure", True, True, False) - return True - - def unpack(self): - """ - execute the unpack phase. - """ - if self.setup_is_for_src: - self.setup_distfiles() - if self.userpriv: - try: - os.chown(self.env["WORKDIR"], portage_uid, -1) - except OSError, oe: - raise format.GenericBuildError( - "failed forcing %i uid for WORKDIR: %s" % - (portage_uid, str(oe))) - return self._generic_phase("unpack", True, True, False) - - compile = pretty_docs( - observer.decorate_build_method("compile")( - post_curry( - ebd._generic_phase, "compile", True, True, False)), - "run the compile phase (maps to src_compile)") - - @observer.decorate_build_method("install") - @_reset_env_data_source - def install(self): - """run the install phase (maps to src_install)""" - if self.fakeroot: - return self._generic_phase("install", True, False, True) - else: - return self._generic_phase("install", False, True, False) - - @observer.decorate_build_method("test") - @_reset_env_data_source - def test(self): - """run the test phase (if enabled), maps to src_test""" - if not self.run_test: - return True - return self._generic_phase("test", True, True, False) - - def finalize(self): - """ - finalize the operation. - - this yields a built package, but the packages - metadata/contents are bound to the workdir. In other words, - install the package somewhere prior to executing clean if you - intend on installing it. - - @return: L{pkgcore.ebuild.ebuild_built.package} instance - """ - return fake_package_factory(self._built_class).new_package(self.pkg, - self.env["IMAGE"], pjoin(self.env["T"], "environment")) - - -class binpkg_buildable(ebd, setup_mixin, format.build): - - stage_depends = {"finalize":"setup", "setup":"start"} - setup_is_for_src = False - - def __init__(self, *args, **kwargs): - ebd.__init__(self, *args, **kwargs) - - def finalize(self): - return MutatedPkg(self.pkg, {"environment":self.get_env_source()}) diff --git a/pkgcore/ebuild/ebuild_built.py b/pkgcore/ebuild/ebuild_built.py deleted file mode 100644 index b91e5c5..0000000 --- a/pkgcore/ebuild/ebuild_built.py +++ /dev/null @@ -1,224 +0,0 @@ -# Copyright: 2005-2007 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -""" -built ebuild packages (vdb packages and binpkgs are derivatives of this) -""" - -from pkgcore.ebuild import ebuild_src -from pkgcore.package import metadata -from pkgcore.interfaces.data_source import local_source - -from snakeoil.mappings import IndeterminantDict -from snakeoil.currying import post_curry -from snakeoil.obj import DelayedInstantiation - -from snakeoil.demandload import demandload -demandload(globals(), - 'pkgcore.merge:engine', - 'pkgcore.ebuild:triggers', - 're', - 'pkgcore.ebuild:ebd', - 'pkgcore.fs.livefs:scan', -) - - -def passthrough(inst, attr, rename=None): - if rename is None: - rename = attr - return inst.data[rename] - -def flatten_depset(inst, conditionals): - return inst.evaluate_depset(conditionals) - -default_pkg_preinst_re = None - -def pkg_uses_default_preinst(pkg): - global default_pkg_preinst_re - if default_pkg_preinst_re is None: - default_pkg_preinst_re = re.compile( - "(?:^|\n)pkg_preinst *\(\)\s*{\s*return;?\s*}[ \t]*(?:\n|$)") - - data = pkg.environment.get_fileobj().read() - m = default_pkg_preinst_re.search(data) - - # second check. make sure there aren't two matches- if so, that - # means we should not guess it. - return m is not None and \ - default_pkg_preinst_re.search(data[m.end():]) is None - -def wrap_inst(self, wrap, inst): - return wrap(inst(self), self.use) - -class package(ebuild_src.base): - - """ - built form of an ebuild - """ - - immutable = True - tracked_attributes = list(ebuild_src.package.tracked_attributes) - tracked_attributes.extend(["contents", "use", "environment"]) - tracked_attributes = tuple(tracked_attributes) - allow_regen = False - - built = True - - _get_attr = dict(ebuild_src.package._get_attr) - - del _get_attr["fetchables"] - - _get_attr.update((x, post_curry(passthrough, x)) - for x in ("contents", "environment", "ebuild")) - _get_attr.update( - (k, post_curry(wrap_inst, - ebuild_src.package._config_wrappables[k], - ebuild_src.package._get_attr[k])) - for k in ebuild_src.package._config_wrappables - if k in ebuild_src.package.tracked_attributes) - - _get_attr["use"] = lambda s:DelayedInstantiation(frozenset, - lambda: frozenset(s.data["USE"].split())) - - def _update_metadata(self, pkg): - raise NotImplementedError() - - def _repo_install_op(self, *args, **kwds): - return self._parent._generate_format_install_op(self, *args, **kwds) - - def _repo_uninstall_op(self, *args, **kwds): - return self._parent._generate_format_uninstall_op(self, *args, **kwds) - - def _repo_replace_op(self, *args, **kwds): - return self._parent._generate_format_replace_op(self, *args, **kwds) - - def _fetch_metadata(self): - return self._parent._get_metadata(self) - - def __str__(self): - return "built ebuild: %s" % (self.cpvstr) - - def build(self, **kwargs): - return self.repo._generate_build_op(self) - - def add_format_triggers(self, *args, **kwds): - return self._parent._add_format_triggers(self, *args, **kwds) - - @property - def ebuild(self): - o = self.data.get("ebuild") - if o is not None: - return o - return self._parent._get_ebuild_src(self) - - @property - def _mtime_(self): - raise AttributeError(self, "_mtime_") - - -def generic_format_triggers(self, pkg, op_inst, format_op_inst, engine_inst): - if (engine_inst.mode in (engine.REPLACE_MODE, engine.INSTALL_MODE) - and pkg == engine_inst.new and pkg.repo is engine_inst.new.repo): - if not pkg_uses_default_preinst(pkg): - t = triggers.preinst_contents_reset(format_op_inst) - t.register(engine_inst) - # for ebuild format, always check the syms. - # this isn't perfect for binpkgs since if the binpkg is already - # screwed, the target is in place already - triggers.FixImageSymlinks(format_op_inst).register(engine_inst) - -def _generic_format_install_op(self, pkg, domain_settings, **kwds): - return ebd.install_op(pkg, initial_env=domain_settings, - env_data_source=pkg.environment, **kwds) - -def _generic_format_uninstall_op(self, pkg, domain_settings, **kwds): - return ebd.uninstall_op(pkg, initial_env=domain_settings, - env_data_source=pkg.environment, **kwds) - -def _generic_format_replace_op(self, pkg, domain_settings, **kwds): - return ebd.replace_op(pkg, initial_env=domain_settings, - env_data_source=pkg.environment, **kwds) - - -class package_factory(metadata.factory): - child_class = package - - # For the plugin system. - priority = 5 - - def _get_metadata(self, pkg): - return self._parent_repo._get_metadata(pkg) - - def new_package(self, *args): - inst = self._cached_instances.get(args) - if inst is None: - inst = self._cached_instances[args] = self.child_class(self, *args) - return inst - - _generate_format_install_op = _generic_format_install_op - _generate_format_uninstall_op = _generic_format_uninstall_op - _generate_format_replace_op = _generic_format_replace_op - _add_format_triggers = generic_format_triggers - - -class fake_package_factory(package_factory): - """ - a fake package_factory, so that we can reuse the normal get_metadata hooks. - - a factory is generated per package instance, rather then one - factory, N packages. - - Do not use this unless you know it's what your after; this is - strictly for transitioning a built ebuild (still in the builddir) - over to an actual repo. It literally is a mapping of original - package data to the new generated instances data store. - """ - - def __init__(self, child_class): - self.child_class = child_class - self._parent_repo = None - - def __del__(self): - pass - - _forced_copy = ebuild_src.package.tracked_attributes - - def new_package(self, pkg, image_root, environment_path): - self.pkg = pkg - self.image_root = image_root - self.environment_path = environment_path - # lambda redirects path to environment path - obj = self.child_class(self, pkg.cpvstr) - for x in self._forced_copy: - # bypass setattr restrictions. - object.__setattr__(obj, x, getattr(self.pkg, x)) - object.__setattr__(obj, "use", self.pkg.use) - return obj - - def get_ebuild_src(self, pkg): - return self.pkg.ebuild - - def scan_contents(self, location): - return scan(location, offset=location, mutable=True) - - def _get_metadata(self, pkg): - return IndeterminantDict(self.__pull_metadata) - - def __pull_metadata(self, key): - if key == "contents": - return self.scan_contents(self.image_root) - elif key == "environment": - return local_source(self.environment_path) - else: - try: - return getattr(self.pkg, key) - except AttributeError: - raise KeyError - - _generate_format_install_op = _generic_format_install_op - _generate_format_uninstall_op = _generic_format_uninstall_op - _generate_format_replace_op = _generic_format_replace_op - _add_format_triggers = generic_format_triggers - - -generate_new_factory = package_factory diff --git a/pkgcore/ebuild/ebuild_src.py b/pkgcore/ebuild/ebuild_src.py deleted file mode 100644 index b892a37..0000000 --- a/pkgcore/ebuild/ebuild_src.py +++ /dev/null @@ -1,367 +0,0 @@ -# Copyright: 2005-2007 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -""" -package class for buildable ebuilds -""" - -import os -from itertools import imap - -from pkgcore.package import metadata -from pkgcore.package import errors as metadata_errors -from pkgcore.ebuild.cpv import CPV -from pkgcore.ebuild import conditionals -from pkgcore.ebuild.atom import atom -from pkgcore.cache import errors as cache_errors -from pkgcore.restrictions.packages import AndRestriction -from pkgcore.restrictions import boolean -from pkgcore.chksum.errors import MissingChksum -from pkgcore.fetch.errors import UnknownMirror -from pkgcore.fetch import fetchable, mirror, uri_list, default_mirror -from pkgcore.ebuild import const, processor - -from snakeoil.mappings import IndeterminantDict -from snakeoil.currying import alias_class_method, partial - -from snakeoil.demandload import demandload -demandload(globals(), "pkgcore.log:logger") - -WeakValCache = metadata.WeakValCache - -def generate_depset(c, key, non_package_type, s, **kwds): - try: - if non_package_type: - return conditionals.DepSet(s.data.pop(key, ""), c, - operators={"||":boolean.OrRestriction, - "":boolean.AndRestriction}, **kwds) - return conditionals.DepSet(s.data.pop(key, ""), c, **kwds) - except conditionals.ParseError, p: - raise metadata_errors.MetadataException(s, str(key), str(p)) - -def generate_providers(self): - rdep = AndRestriction(self.versioned_atom, finalize=True) - func = partial(virtual_ebuild, self._parent, self, - {"rdepends":rdep, "slot":"%s-%s" % (self.category, self.version)}) - # re-enable license at some point. - #, "license":self.license}) - - try: - return conditionals.DepSet( - self.data.pop("PROVIDE", ""), virtual_ebuild, element_func=func, - operators={"":boolean.AndRestriction}) - - except conditionals.ParseError, p: - raise metadata_errors.MetadataException(self, "provide", str(p)) - -def generate_fetchables(self): - chksums = self.repo._get_digests(self) - - mirrors = getattr(self._parent, "mirrors", {}) - default_mirrors = getattr(self._parent, "default_mirrors", None) - common = {} - try: - d = conditionals.DepSet( - self.data.pop("SRC_URI", ""), fetchable, operators={}, - element_func=partial(create_fetchable_from_uri, self, chksums, - mirrors, default_mirrors, common)) - for v in common.itervalues(): - v.uri.finalize() - return d - except conditionals.ParseError, p: - raise metadata_errors.MetadataException(self, "src_uri", str(p)) - -# utility func. -def create_fetchable_from_uri(pkg, chksums, mirrors, default_mirrors, - common_files, uri): - - filename = os.path.basename(uri) - - preexisting = common_files.get(filename) - - if preexisting is None: - if filename not in chksums: - raise MissingChksum(filename) - uris = uri_list(filename) - else: - uris = preexisting.uri - - if filename != uri: - if preexisting is None: - if "primaryuri" not in pkg.restrict: - if default_mirrors and "mirror" not in pkg.restrict: - uris.add_mirror(default_mirrors) - - if uri.startswith("mirror://"): - # mirror:// is 9 chars. - - tier, remaining_uri = uri[9:].split("/", 1) - - if tier not in mirrors: - raise UnknownMirror(tier, remaining_uri) - - uris.add_mirror(mirrors[tier], remaining_uri) - - else: - uris.add_uri(uri) - if preexisting is None and "primaryuri" in pkg.restrict: - if default_mirrors and "mirror" not in pkg.restrict: - uris.add_mirror(default_mirrors) - - if preexisting is None: - common_files[filename] = fetchable(filename, uris, chksums[filename]) - return common_files[filename] - -def generate_eapi(self): - try: - d = self.data.pop("EAPI", 0) - if d == "": - return 0 - return int(d) - except ValueError: - return const.unknown_eapi - -def get_slot(self): - o = self.data.pop("SLOT", "0").strip() - if not o: - raise ValueError(self, "SLOT cannot be unset") - return o - -def rewrite_restrict(restrict): - if restrict[0:2] == 'no': - return restrict[2:] - return restrict - - -class base(metadata.package): - - """ - ebuild package - - @cvar tracked_attributes: sequence of attributes that are required to exist - in the built version of ebuild-src - @cvar _config_wrappables: mapping of attribute to callable for - re-evaluating attributes dependant on configuration - """ - - tracked_attributes = ( - "depends", "rdepends", "post_rdepends", "provides", "license", - "slot", "keywords", "eapi", "restrict", "eapi", "description", "iuse") - - _config_wrappables = dict((x, alias_class_method("evaluate_depset")) - for x in ["depends", "rdepends", "post_rdepends", "fetchables", - "license", "src_uri", "license", "provides", "restrict"]) - - _get_attr = dict(metadata.package._get_attr) - _get_attr["provides"] = generate_providers - _get_attr["depends"] = partial(generate_depset, atom, "DEPEND", False) - _get_attr["rdepends"] = partial(generate_depset, atom, "RDEPEND", False) - _get_attr["post_rdepends"] = partial(generate_depset, atom, "PDEPEND", - False) - _get_attr["license"] = partial(generate_depset, str, - "LICENSE", True, element_func=intern) - _get_attr["slot"] = get_slot # lambda s: s.data.pop("SLOT", "0").strip() - _get_attr["fetchables"] = generate_fetchables - _get_attr["description"] = lambda s:s.data.pop("DESCRIPTION", "").strip() - _get_attr["keywords"] = lambda s:tuple(map(intern, - s.data.pop("KEYWORDS", "").split())) - _get_attr["restrict"] = lambda s:conditionals.DepSet( - s.data.pop("RESTRICT", ''), str, operators={}, - element_func=rewrite_restrict) - _get_attr["eapi"] = generate_eapi - _get_attr["iuse"] = lambda s:frozenset(imap(intern, - s.data.pop("IUSE", "").split())) - _get_attr["homepage"] = lambda s:s.data.pop("HOMEPAGE", "").strip() - - __slots__ = tuple(_get_attr.keys() + ["_pkg_metadata_shared"]) - - @property - def P(self): - return "%s-%s" % (self.package, self.version) - - @property - def PF(self): - return "%s-%s" % (self.package, self.fullver) - - @property - def PN(self): - return self.package - - @property - def PR(self): - r = self.revision - if r is not None: - return r - return 0 - - @property - def ebuild(self): - return self._parent.get_ebuild_src(self) - - def _fetch_metadata(self): - return self._parent._get_metadata(self) - - def __str__(self): - return "ebuild src: %s" % self.cpvstr - - def __repr__(self): - return "<%s cpv=%r @%#8x>" % (self.__class__, self.cpvstr, id(self)) - - -class package(base): - - __slots__ = ("_shared_pkg_data") - - _get_attr = dict(base._get_attr) - - def __init__(self, shared_pkg_data, *args, **kwargs): - base.__init__(self, *args, **kwargs) - object.__setattr__(self, "_shared_pkg_data", shared_pkg_data) - - @property - def maintainers(self): - return self._shared_pkg_data.metadata_xml.maintainers - - @property - def herds(self): - return self._shared_pkg_data.metadata_xml.herds - - @property - def longdescription(self): - return self._shared_pkg_data.metadata_xml.longdescription - - @property - def _mtime_(self): - return self._parent._get_ebuild_mtime(self) - - @property - def manifest(self): - return self._shared_pkg_data.manifest - - -class package_factory(metadata.factory): - child_class = package - - # For the plugin system. - priority = 5 - - def __init__(self, parent, cachedb, eclass_cache, mirrors, default_mirrors, - *args, **kwargs): - super(package_factory, self).__init__(parent, *args, **kwargs) - self._cache = cachedb - self._ecache = eclass_cache - if mirrors: - mirrors = dict((k, mirror(v, k)) for k, v in mirrors.iteritems()) - - self.mirrors = mirrors - if default_mirrors: - self.default_mirrors = default_mirror(default_mirrors, - "conf. default mirror") - else: - self.default_mirrors = None - - def get_ebuild_src(self, pkg): - return self._parent_repo._get_ebuild_src(pkg) - - def _get_ebuild_path(self, pkg): - return self._parent_repo._get_ebuild_path(pkg) - - def _get_ebuild_mtime(self, pkg): - return os.stat(self._get_ebuild_path(pkg)).st_mtime - - def _invalidated_eclasses(self, data, pkg): - return (data.get("_eclasses_") is not None and not - self._ecache.is_eclass_data_valid(data["_eclasses_"])) - - def _get_metadata(self, pkg): - for cache in self._cache: - if cache is not None: - try: - data = cache[pkg.cpvstr] - except KeyError: - continue - except cache_errors.CacheError, ce: - logger.warn("caught cache error: %s" % ce) - del ce - continue - if long(data.pop("_mtime_", -1)) != long(pkg._mtime_) or \ - self._invalidated_eclasses(data, pkg): - if not cache.readonly: - del cache[pkg.cpvstr] - continue - return data - - # no cache entries, regen - return self._update_metadata(pkg) - - def _update_metadata(self, pkg): - ebp = processor.request_ebuild_processor() - try: - mydata = ebp.get_keys(pkg, self._ecache) - finally: - processor.release_ebuild_processor(ebp) - - mydata["_mtime_"] = pkg._mtime_ - if mydata.get("INHERITED", False): - mydata["_eclasses_"] = self._ecache.get_eclass_data( - mydata["INHERITED"].split()) - del mydata["INHERITED"] - else: - mydata["_eclasses_"] = {} - - if self._cache is not None: - for cache in self._cache: - if not cache.readonly: - cache[pkg.cpvstr] = mydata - break - - return mydata - - def new_package(self, *args): - inst = self._cached_instances.get(args) - if inst is None: - # key being cat/pkg - mxml = self._parent_repo._get_shared_pkg_data(args[0], args[1]) - inst = self._cached_instances[args] = self.child_class( - mxml, self, *args) - return inst - - -generate_new_factory = package_factory - - -class virtual_ebuild(metadata.package): - - """ - PROVIDES generated fake packages - """ - - package_is_real = False - built = True - - __slots__ = ("_orig_data", "data", "provider") - - def __init__(self, parent_repository, pkg, data, cpvstr): - """ - @param cpvstr: cpv for the new pkg - @param parent_repository: actual repository that this pkg should - claim it belongs to - @param pkg: parent pkg that is generating this pkg - @param data: mapping of data to push to use in __getattr__ access - """ - c = CPV(cpvstr) - if c.fullver is None: - cpvstr = cpvstr + "-" + pkg.fullver - - metadata.package.__init__(self, parent_repository, cpvstr) - sfunc = object.__setattr__ - sfunc(self, "data", IndeterminantDict(lambda *a: str(), data)) - sfunc(self, "_orig_data", data) - sfunc(self, "provider", pkg.versioned_atom) - - def __getattr__(self, attr): - if attr in self._orig_data: - return self._orig_data[attr] - return metadata.package.__getattr__(self, attr) - - _get_attr = package._get_attr.copy() diff --git a/pkgcore/ebuild/eclass_cache.py b/pkgcore/ebuild/eclass_cache.py deleted file mode 100644 index 523ec80..0000000 --- a/pkgcore/ebuild/eclass_cache.py +++ /dev/null @@ -1,135 +0,0 @@ -# Copyright: 2005-2006 Brian Harring <ferringb@gmail.com> -# Based upon eclass_cache from portage; matches api, but was reimplemented. -# License: GPL2 - -""" -in memory representation of on disk eclass stacking order -""" - -from pkgcore.interfaces.data_source import local_source -from pkgcore.config import ConfigHint - -from snakeoil.mappings import ImmutableDict -from snakeoil.weakrefs import WeakValCache -from snakeoil.osutils import join as pjoin - -from snakeoil.demandload import demandload -demandload(globals(), - "os", - "snakeoil.osutils:normpath", - "snakeoil.mappings:StackedDict", -) - -class base(object): - """ - Maintains the cache information about eclasses available to an ebuild. - """ - - def __init__(self, portdir=None, eclassdir=None): - self._eclass_data_inst_cache = WeakValCache() - # generate this. - # self.eclasses = {} # {"Name": ("location","_mtime_")} - self.eclasses = {} - self.portdir = portdir - self.eclassdir = eclassdir - - def is_eclass_data_valid(self, ec_dict): - """Check if eclass data is still valid. - - Given a dict as returned by get_eclass_data, walk it comparing - it to internal eclass view. - - @return: a boolean representing whether that eclass data is still - up to date, or not - """ - ec = self.eclasses - for eclass, tup in ec_dict.iteritems(): - if eclass not in ec: - return False - elif isinstance(tup, tuple): - if tup[1] != ec[eclass][1]: - return False - elif tup != ec[eclass][1]: - return False - return True - - def get_eclass_data(self, inherits): - """Return the cachable entries from a list of inherited eclasses. - - Only make get_eclass_data calls for data you know came from - this eclass_cache, otherwise be ready to catch a KeyError - exception for any eclass that was requested, but not known to - this cache. - """ - - keys = tuple(sorted(inherits)) - o = self._eclass_data_inst_cache.get(keys) - if o is None: - o = ImmutableDict((k, self.eclasses[k]) for k in keys) - self._eclass_data_inst_cache[keys] = o - return o - - def get_eclass(self, eclass): - o = self.eclasses.get(eclass) - if o is None: - return None - return local_source(pjoin(o[0], eclass+".eclass")) - - -class cache(base): - - pkgcore_config_type = ConfigHint({"path":"str", "portdir":"str"}, - typename='eclass_cache') - - def __init__(self, path, portdir=None): - """ - @param portdir: ondisk location of the tree we're working with - """ - base.__init__(self, portdir=portdir, eclassdir=normpath(path)) - self.update_eclasses() - - def update_eclasses(self): - """Force an update of the internal view of on disk/remote eclasses.""" - self.eclasses = {} - eclass_len = len(".eclass") - if os.path.isdir(self.eclassdir): - for y in os.listdir(self.eclassdir): - if not y.endswith(".eclass"): - continue - try: - mtime = os.stat(pjoin(self.eclassdir, y)).st_mtime - except OSError: - continue - ys = y[:-eclass_len] - self.eclasses[intern(ys)] = (self.eclassdir, long(mtime)) - - -class StackedCaches(cache): - - """ - collapse multiple eclass caches into one. - - Does L->R searching for eclass matches. - """ - - pkgcore_config_type = ConfigHint( - {'caches': 'refs:eclass_cache', 'portdir': 'str', 'eclassdir': 'str'}, - typename='eclass_cache') - - def __init__(self, caches, **kwds): - """ - @param caches: L{cache} instances to stack; - ordering should be desired lookup order - @keyword eclassdir: override for the master eclass dir, required for - eapi0 and idiot eclass usage. defaults to pulling from the first - cache. - """ - if len(caches) < 2: - raise TypeError( - "%s requires at least two eclass_caches" % self.__class__) - - kwds.setdefault("eclassdir", caches[0].eclassdir) - kwds.setdefault("portdir", - os.path.dirname(kwds["eclassdir"].rstrip(os.path.sep))) - base.__init__(self, **kwds) - self.eclasses = StackedDict(*[ec.eclasses for ec in caches]) diff --git a/pkgcore/ebuild/errors.py b/pkgcore/ebuild/errors.py deleted file mode 100644 index 17b6a90..0000000 --- a/pkgcore/ebuild/errors.py +++ /dev/null @@ -1,53 +0,0 @@ -# Copyright: 2005 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -# "More than one statement on a single line" -# pylint: disable-msg=C0321 - -""" -atom exceptions -""" - -from pkgcore.package import errors - -class MalformedAtom(errors.InvalidDependency): - - def __init__(self, atom, err=''): - errors.InvalidDependency.__init__( - self, "atom '%s' is malformed: error %s" % (atom, err)) - self.atom, self.err = atom, err - - -class InvalidVersion(errors.InvalidDependency): - - def __init__(self, ver, rev, err=''): - errors.InvalidDependency.__init__( - self, - "Version restriction ver='%s', rev='%s', is malformed: error %s" % - (ver, rev, err)) - self.ver, self.rev, self.err = ver, rev, err - - -class InvalidCPV(errors.InvalidPackage): - """Raised if an invalid cpv was passed in. - - @ivar args: single-element tuple containing the invalid string. - @type args: C{tuple} - """ - - -class ParseError(errors.InvalidDependency): - - def __init__(self, s, token=None, msg=None): - if msg is None: - str_msg = '' - else: - str_msg = ': %s' % msg - if token is not None: - Exception.__init__(self, - "%s is unparseable%s\nflagged token- %s" % - (s, str_msg, token)) - else: - Exception.__init__(self, - "%s is unparseable%s" % (s, str_msg)) - self.dep_str, self.token, self.msg = s, token, msg diff --git a/pkgcore/ebuild/filter_env.py b/pkgcore/ebuild/filter_env.py deleted file mode 100644 index cc8ca55..0000000 --- a/pkgcore/ebuild/filter_env.py +++ /dev/null @@ -1,418 +0,0 @@ -# Copyright: 2006 Marien Zwart <marienz@gentoo.org> -# Copyright: 2007 Brian Harring <ferringb@gmail.com> -# License: GPL2 -# Based on filter-env by Brian Harring <ferringb@gmail.com> -# and Mike Frysinger <spanky@gentoo.org> - - -"""Filter a bash environment dump.""" - - -from snakeoil.demandload import demandload -demandload(globals(), - 're', - 'pkgcore.log:logger' -) - - -COMMAND_PARSING, SPACE_PARSING = range(2) - - -def native_run(out, file_buff, vsr, fsr, - desired_var_match, desired_func_match): - """Print a filtered environment. - - @param out: file-like object to write to. - @param file_buff: string containing the environment to filter. - Should end in '\0'. - @param vsr: result of build_regex_string or C{None}, for variables. - @param vsr: result of build_regex_string or C{None}, for functions. - @param desired_var_match: boolean indicating vsr should match or not. - @param desired_func_match: boolean indicating fsr should match or not. - """ - if fsr is None: - func_match = None - else: - fsr = re.compile(fsr) - if desired_func_match: - func_match = fsr.match - else: - def func_match(data): - return fsr.match(data) is None - - if vsr is None: - var_match = None - else: - vsr = re.compile(vsr) - if desired_var_match: - var_match = vsr.match - else: - def var_match(data): - return vsr.match(data) is None - - process_scope(out, file_buff, 0, var_match, func_match, '\0') - - -try: - from pkgcore.ebuild._filter_env import run -except ImportError: - cpy_run = None - run = native_run -else: - cpy_run = run - - -def build_regex_string(tokens): - if not tokens: - return None - result = [] - for token in tokens: - if not token: - continue - escaped = False - l = [] - for ch in token: - if ch == '.' and not escaped: - l.append('[^= ]') - else: - l.append(ch) - if ch == '\\': - escaped = not escaped - else: - escaped = False - result.append(''.join(l)) - if len(result) == 1: - return '^%s$' % result[0] - return '^(%s)$' % '|'.join(result) - - -FUNC_LEN = len('function') -def is_function(buff, pos): - """@returns: start, end, pos or None, None, None tuple.""" - isspace = str.isspace - while buff[pos] in ' \t': - pos += 1 - if buff[pos:pos + FUNC_LEN] == 'function': - pos += FUNC_LEN - while isspace(buff[pos]): - pos += 1 - start = pos - while buff[pos] not in '\0 \t\n="\'()': - pos += 1 - end = pos - if end == start: - return None, None, None - while buff[pos] in ' \t': - pos += 1 - if buff[pos] != '(': - return None, None, None - pos += 1 - while buff[pos] in ' \t': - pos += 1 - if buff[pos] != ')': - return None, None, None - pos += 1 - while isspace(buff[pos]): - pos += 1 - if buff[pos] != '{': - return None, None, None - return start, end, pos + 1 - - -def is_envvar(buff, pos): - """@returns: start, end, pos or None, None, None tuple.""" - while buff[pos] in ' \t': - pos += 1 - start = pos - while True: - if buff[pos] in '\0"\'()- \t\n': - return None, None, None - if buff[pos] == '=': - if pos == start: - return None, None, None - return start, pos, pos + 1 - pos += 1 - - -def process_scope(out, buff, pos, var_match, func_match, endchar): - window_start = pos - window_end = None - isspace = str.isspace - end = len(buff) - while pos < end and buff[pos] != endchar: - # Wander forward to the next non space. - if window_end is not None: - if out is not None: - out.write(buff[window_start:window_end]) - window_start = pos - window_end = None - com_start = pos - ch = buff[pos] - if isspace(ch): - pos += 1 - continue - - # Ignore comments. - if ch == '#': - pos = walk_statement_pound(buff, pos, endchar) - continue - - new_start, new_end, new_p = is_function(buff, pos) - if new_p is not None: - func_name = buff[new_start:new_end] - logger.debug('matched func name %r', func_name) - new_p = process_scope(None, buff, new_p, None, None, '}') - logger.debug('ended processing %r', func_name) - if func_match is not None and func_match(func_name): - logger.debug('filtering func %r', func_name) - window_end = com_start - pos = new_p - pos += 1 - continue - # Check for env assignment. - new_start, new_end, new_p = is_envvar(buff, pos) - if new_p is None: - # Non env assignment. - pos = walk_command_complex(buff, pos, endchar, COMMAND_PARSING) - # icky icky icky icky - if pos < end and buff[pos] != endchar: - pos += 1 - else: - # Env assignment. - var_name = buff[new_start:new_end] - pos = new_p - logger.debug('matched env assign %r', var_name) - - if var_match is not None and var_match(var_name): - # This would be filtered. - logger.info("filtering var '%s'", var_name) - window_end = com_start - - if pos >= end: - return pos - - while (pos < end and not isspace(buff[pos]) - and buff[pos] != ';'): - if buff[pos] == "'": - pos = walk_statement_no_parsing(buff, pos + 1, "'") + 1 - elif buff[pos] in '"`': - pos = walk_command_escaped_parsing(buff, pos + 1, - buff[pos]) + 1 - elif buff[pos] == '(': - pos = walk_command_escaped_parsing(buff, pos + 1, ')') + 1 - elif buff[pos] == '$': - pos += 1 - if pos >= end: - continue - pos = walk_dollar_expansion(buff, pos, end, endchar) - continue - else: - # blah=cah ; single word - pos = walk_command_complex(buff, pos, ' ', SPACE_PARSING) - - if out is not None: - if window_end is None: - window_end = pos - if window_end > end: - window_end = end - out.write(buff[window_start:window_end]) - - return pos - - -def walk_statement_no_parsing(buff, pos, endchar): - pos = buff.find(endchar, pos) - if pos == -1: - pos = len(buff) - 1 - return pos - - -def walk_statement_dollared_quote_parsing(buff, pos, endchar): - end = len(buff) - while pos < end: - if buff[pos] == endchar: - return pos - elif buff[pos] == '\\': - pos += 1 - pos += 1 - return pos - - -def walk_here_statement(buff, pos): - pos += 1 - logger.debug('starting here processing for COMMAND for level 2 at p == %.10s', - pos) - if buff[pos] == '<': - logger.debug( - "correction, it's a third level here. Handing back to command " - 'parsing') - return pos + 1 - isspace = str.isspace - end = len(buff) - while pos < end and (isspace(buff[pos]) or buff[pos] == '-'): - pos += 1 - if buff[pos] in "'\"": - end_here = walk_statement_no_parsing(buff, pos + 1, buff[pos]) - pos += 1 - else: - end_here = walk_command_complex(buff, pos, ' ', SPACE_PARSING) - here_word = buff[pos:end_here] - logger.debug('matched len(%s)/%r for a here word', - len(here_word), here_word) - # XXX watch this. Potential for horkage. Need to do the quote - # removal thing. This sucks. - end_here += 1 - if end_here >= end: - return end_here - - here_len = len(here_word) - end_here = buff.find(here_word, end_here) - while end_here != -1: - i = here_len + end_here - if buff[i] in ';\n\r})': - i = end_here - 1 - while i >= 0 and buff[i] in '\t ': - i -= 1 - if i >= 0 and buff[i] == '\n': - break - end_here = buff.find(here_word, end_here + here_len) - - if end_here == -1: - return end - return end_here + len(here_word) - - -def walk_statement_pound(buff, pos, endchar=None): - if pos and not buff[pos-1].isspace(): - return pos + 1 - if endchar == '`': - i = buff.find('\n', pos) - i2 = buff.find(endchar, pos) - if i == -1: - if i2 != -1: - return i2 - else: - if i2 != -1: - return min(i, i2) - return i - return len(buff) - 1 - - pos = buff.find('\n', pos) - if pos == -1: - pos = len(buff) - 1 - return pos - - -def walk_command_complex(buff, pos, endchar, interpret_level): - start = pos - isspace = str.isspace - end = len(buff) - while pos < end: - ch = buff[pos] - if ch == endchar: - if endchar != '}': - return pos - if start == pos: - return pos - if buff[pos - 1] in ";\n": - return pos - elif (interpret_level == COMMAND_PARSING and ch in ';\n') or \ - (interpret_level == SPACE_PARSING and isspace(ch)): - return pos - elif ch == '\\': - pos += 1 - elif ch == '<': - if (pos < end - 1 and buff[pos + 1] == '<' and - interpret_level == COMMAND_PARSING): - pos = walk_here_statement(buff, pos + 1) - # we continue immediately; walk_here deposits us at the end - # of the here op, not consuming the final delimiting char - # since it may be an endchar - continue - else: - logger.debug('noticed <, interpret_level=%s', interpret_level) - elif ch == '#': - if start == pos or isspace(buff[pos - 1]) or buff[pos - 1] == ';': - pos = walk_statement_pound(buff, pos) - continue - elif ch == '$': - pos = walk_dollar_expansion(buff, pos + 1, end, endchar) - continue - elif ch == '{': - pos = walk_command_escaped_parsing(buff, pos + 1, '}') - elif ch == '(' and interpret_level == COMMAND_PARSING: - pos = walk_command_escaped_parsing(buff, pos + 1, ')') - elif ch in '`"': - pos = walk_command_escaped_parsing(buff, pos + 1, ch) - elif ch == "'" and endchar != '"': - pos = walk_statement_no_parsing(buff, pos +1, "'") - pos += 1 - return pos - -def raw_walk_command_escaped_parsing(buff, pos, endchar): - end = len(buff) - while pos < end: - ch = buff[pos] - if ch == endchar: - return pos - elif ch == '\\': - pos += 1 - elif ch == '{': - if endchar != '"': - pos = raw_walk_command_escaped_parsing( - buff, pos + 1, '}') - elif ch == '(': - if endchar != '"': - pos = raw_walk_command_escaped_parsing( - buff, pos + 1, ')') - elif ch in '`"': - pos = raw_walk_command_escaped_parsing(buff, pos + 1, ch) - elif ch == "'" and endchar != '"': - pos = walk_statement_no_parsing(buff, pos + 1, "'") - elif ch == '$': - pos = walk_dollar_expansion(buff, pos + 1, end, endchar, - disable_quote = endchar == '"') - continue - elif ch == '#' and endchar != '"': - pos = walk_statement_pound(buff, pos, endchar) - continue - pos += 1 - return pos - -walk_command_escaped_parsing = raw_walk_command_escaped_parsing - -def walk_dollar_expansion(buff, pos, end, endchar, disable_quote=False): - if buff[pos] == '(': - return process_scope(None, buff, pos + 1, None, None, ')') + 1 - if buff[pos] == "'" and not disable_quote: - return walk_statement_dollared_quote_parsing(buff, pos +1, "'") + 1 - if buff[pos] != '{': - if buff[pos] == '$': - # short circuit it. - return pos + 1 - while pos < end and buff[pos] != endchar: - if buff[pos].isspace(): - return pos - if buff[pos] == '$': - # shouldn't this be passing disable_quote ? - return walk_dollar_expansion(buff, pos + 1, end, endchar) - if not buff[pos].isalnum(): - if buff[pos] != '_': - return pos - pos += 1 - - if pos >= end: - return end - return pos - - pos += 1 - # shortcut ${$} to avoid going too deep. ${$a} isn't valid, so no concern - if pos == '$': - return pos + 1 - while pos < end and buff[pos] != '}': - if buff[pos] == '$': - # disable_quote? - pos = walk_dollar_expansion(buff, pos + 1, end, endchar) - else: - pos += 1 - return pos + 1 diff --git a/pkgcore/ebuild/formatter.py b/pkgcore/ebuild/formatter.py deleted file mode 100644 index 436706e..0000000 --- a/pkgcore/ebuild/formatter.py +++ /dev/null @@ -1,486 +0,0 @@ -# Copyright: 2006 Charlie Shepherd <masterdriverz@gentoo.org> -# License: GPL2 - -"""PMerge formatting module - -To add a new formatter, add the relevant class (which -should be a subclass of Formatter). Documentation is -a necessity - things can change/break easily between -versions. Then add the class name (_not_ an instance) to -the formatters dictionary - this will instantly make your -formatter available on the commandline. -""" - -import operator - -from pkgcore.config import configurable -from snakeoil.demandload import demandload -demandload(globals(), 'errno') - -class NoChoice(KeyboardInterrupt): - """Raised by L{userquery} if no choice was made. - - HACK: this subclasses KeyboardInterrupt, so if you ignore this it - should do something reasonable. - """ - -def userquery(prompt, out, err, responses=None, default_answer=None, limit=3): - """Ask the user to choose from a set of options. - - Displays a prompt and a set of responses, then waits for a - response which is checked against the responses. If there is an - unambiguous match the value is returned. - - If the user does not input a valid response after a number of - tries L{NoChoice} is raised. You can catch this if you want to do - something special. Because it subclasses C{KeyboardInterrupt} - the default behaviour is to abort as if the user hit ctrl+c. - - @type prompt: C{basestring} or a tuple of things to pass to a formatter. - XXX this is a crummy api but I cannot think of a better one supporting - the very common case of wanting just a string as prompt. - @type out: formatter. - @type err: formatter. - @type responses: mapping with C{basestring} keys and tuple values. - @param responses: mapping of user input to function result. - The first item in the value tuple is returned, the rest is passed to - out. - Defaults to:: - { - 'yes': (True, out.fg('green'), 'Yes'), - 'no': (False, out.fg('red'), 'No'), - } - @param default_answer: returned if there is no input - (user just hits enter). Defaults to True if responses is unset, - unused otherwise. - @param limit: number of allowed tries. - """ - if responses is None: - responses = { - 'yes': (True, out.fg('green'), 'Yes'), - 'no': (False, out.fg('red'), 'No'), - } - if default_answer is None: - default_answer = True - if default_answer is not None: - for val in responses.itervalues(): - if val[0] == default_answer: - default_answer_name = val[1:] - for i in xrange(limit): - # XXX see docstring about crummyness - if isinstance(prompt, tuple): - out.write(autoline=False, *prompt) - else: - out.write(prompt, autoline=False) - out.write(' [', autoline=False) - prompts = responses.values() - for choice in prompts[:-1]: - out.write(autoline=False, *choice[1:]) - out.write(out.reset, '/', autoline=False) - out.write(autoline=False, *prompts[-1][1:]) - out.write(out.reset, ']', autoline=False) - if default_answer is not None: - out.write(' (default: ', autoline=False) - out.write(autoline=False, *default_answer_name) - out.write(')', autoline=False) - out.write(': ', autoline=False) - try: - response = raw_input() - except EOFError: - out.write("\nNot answerable: EOF on STDIN") - raise NoChoice() - except IOError, e: - if e.errno == errno.EBADF: - out.write("\nNot answerable: STDIN is either closed, or not readable") - raise NoChoice() - raise - if not response: - return default_answer - results = set( - (key, value) for key, value in responses.iteritems() - if key[:len(response)].lower() == response.lower()) - if not results: - err.write('Sorry, response "%s" not understood.' % (response,)) - elif len(results) > 1: - err.write('Response "%s" is ambiguous (%s)' % ( - response, ', '.join(key for key, val in results))) - else: - return list(results)[0][1][0] - - raise NoChoice() - - -class use_expand_filter(object): - - def __init__(self, use_expand, use_expand_hidden): - """ - @type use_expand: iterable of strings - @param use_expand: names of use-expanded variables. - @type use_expand_hidden: set of strings - @param use_expand_hidden: names of use-expanded vars that should not - be added to the dict. - """ - self.expand_filters = dict((x.lower(), (x not in use_expand_hidden, x)) - for x in use_expand) - self.use_expand = use_expand - self.use_expand_hidden = use_expand_hidden - self.known_flags = {} - - def __call__(self, use): - """Split USE flags up into "normal" flags and use-expanded ones. - @type use: iterable of strings - @param use: flags that are set. - @rtype: sequence of strings, dict mapping a string to a list of strings - @return: set of normal flags and a mapping from use_expand name to - value (with the use-expanded bit stripped off, so - C{"video_cards_alsa"} becomes C{"{'video_cards': ['alsa']}"}). - """ - - # XXX: note this is fairly slow- actually takes up more time then chunks of - # the resolver - ue_dict = {} - usel = [] - ef = self.expand_filters - kf = self.known_flags - - for flag in use: - data = kf.get(flag) - if data is None: - split_flag = flag.rsplit("_", 1) - while len(split_flag) == 2: - if split_flag[0] not in ef: - split_flag = split_flag[0].rsplit("_", 1) - continue - expand_state = ef[split_flag[0]] - if expand_state[0]: - # not hidden - kf[flag] = data = (expand_state[1], flag[len(split_flag[0]) + 1:]) - else: - kf[flag] = data = False - break - else: - kf[flag] = data = True - if data is True: - # straight use flag. - usel.append(flag) - elif data: - # non hidden flag. - if not data[0] in ue_dict: - ue_dict[data[0]] = set([data[1]]) - else: - ue_dict[data[0]].add(data[1]) - - return frozenset(usel), ue_dict - - -class Formatter(object): - - """Base Formatter class: All formatters should be subclasses of this.""" - - def __init__(self, **kwargs): - self.__dict__.update(kwargs) - - def format(self, op): - """Formats an op. Subclasses must define this method""" - raise NotImplementedError(self.format) - - def ask(self, question, responses=None, default_answer=None, limit=3): - return userquery( - question, self.out, self.err, responses, default_answer, limit) - - def end(self): - """Called at the end, normally for summary information""" - - -class BasicFormatter(Formatter): - """A basic formatter, intended for scripts""" - def format(self, op): - self.out.write(op.pkg.key) - - -class PkgcoreFormatter(Formatter): - """The original pkgcore output""" - def format(self, op): - repo = getattr(op.pkg.repo, 'repo_id', None) - if not repo: - p = str(op.pkg.cpvstr) - else: - p = "%s::%s" % (op.pkg.cpvstr, repo) - if op.desc == "replace": - self.out.write("replace %s, %s" % (op.old_pkg.cpvstr, p)) - else: - self.out.write("%s %s" % (op.desc.ljust(7), p)) - - -class PortageFormatter(Formatter): - - """Portage formatter - - A Formatter designed to resemble Portage's output - as much as much as possible. - """ - - def __init__(self, **kwargs): - kwargs.setdefault("use_expand", set()) - kwargs.setdefault("use_expand_hidden", set()) - kwargs.setdefault("display_repo", False) - Formatter.__init__(self, **kwargs) - self.use_splitter = use_expand_filter(self.use_expand, - self.use_expand_hidden) - # Map repo location to an index. - self.repos = {} - - def format(self, op): - # [<type> NRFDU] - # <type> - ebuild, block or nomerge (for --tree) - # N - New package - # R - Rebuild package - # F - Fetch restricted - # D - Downgrade - # U - Upgrade - # Caveats: - # - U and D are both displayed to show a downgrade - this is kept - # in order to be consistent with existing portage behaviour - - - out = self.out - origautoline = out.autoline - out.autoline = False - - # This is for the summary at the end - reponr = self.repos.setdefault( - getattr(op.pkg.repo, "repo_id", "<unknown>"), - len(self.repos) + 1) - - # We don't do blockers or --tree stuff yet - out.write('[ebuild ') - - # Order is important here - look at the above diagram - type = op.desc - if op.desc == "add": - out.write(out.fg('green'), ' N') - if op.pkg.slot != '0': - out.write(out.fg('green'), 'S') - else: - out.write(' ') - elif op.desc == "replace" and op.pkg == op.old_pkg: - out.write(out.fg('yellow'), ' R') - else: - out.write(' ') - type = 'upgrade' - - if 'fetch' in op.pkg.restrict: - out.write(out.fg('red'), 'F') - else: - out.write(' ') - if type == 'upgrade': - if op.pkg.fullver != op.old_pkg.fullver: - out.write(out.fg('cyan'), 'U') - if op.pkg > op.old_pkg: - out.write(' ') - else: - out.write(out.fg('blue'), 'D') - else: - out.write(' ') - out.write('] ') - - out.write(out.fg('green'), '%s ' % op.pkg.cpvstr) - - if type == 'upgrade': - out.write(out.fg('blue'), '[%s] ' % op.old_pkg.fullver) - - # Build a list of (useflags, use_expand_dicts) tuples. - # HACK: if we are in "replace" mode we build a list of length - # 4, else this is a list of length 2. We then pass this to - # format_use which can take either 2 or 4 arguments. - if op.desc == 'replace': - uses = (op.pkg.iuse, op.pkg.use, op.old_pkg.iuse, op.old_pkg.use) - else: - uses = (op.pkg.iuse, op.pkg.use) - stuff = map(self.use_splitter, uses) - - # Convert the list of tuples to a list of lists and a list of - # dicts (both length 2 or 4). - uselists, usedicts = zip(*stuff) - self.format_use('use', *uselists) - for expand in self.use_expand-self.use_expand_hidden: - flaglists = [d.get(expand, ()) for d in usedicts] - self.format_use(expand, *flaglists) - - if self.display_repo: - out.write(out.fg('blue'), " [%d]" % (reponr,)) - - out.write('\n') - out.autoline = origautoline - - def format_use(self, attr, selectable, choice, oldselectable=None, - oldchoice=None): - """Write the current selection from a set of flags to a formatter. - - @type attr: string - @param attr: the name of the setting. - @type selectable: set of strings - @param selectable: the possible values. - @type choice: set of strings - @param choice: the chosen values. - @type oldselectable: set of strings - @param oldselectable: the values possible in the previous version. - @type oldchoice: set of strings - @param oldchoice: the previously chosen values. - """ - out = self.out - red = out.fg('red') - green = out.fg('green') - blue = out.fg('blue') - yellow = out.fg('yellow') - - flags = [] - enabled = set(selectable) & set(choice) - disabled = set(selectable) - set(choice) - if oldselectable is not None and oldchoice is not None: - old_enabled = set(oldselectable) & set(oldchoice) - old_disabled = set(oldselectable) - set(oldchoice) - for flag in sorted(enabled): - assert flag - if flag in old_enabled: - # Unchanged flag. - flags.extend((red, flag, ' ')) - elif flag in old_disabled: - # Toggled. - # Trailing single space is important, we can pop it below. - flags.extend((green, flag, '*', ' ')) - else: - # Flag did not exist earlier. - flags.extend((yellow, flag, '%', ' ')) - for flag in sorted(disabled | (set(oldselectable) - set(selectable))): - assert flag - if flag not in disabled: - # Removed flag. - flags.extend((yellow, '(-', flag, '%)', ' ')) - elif flag in old_disabled: - # Unchanged. - flags.extend((blue, '-', flag, ' ')) - elif flag in old_enabled: - # Toggled. - flags.extend((yellow, '-', flag, '*', ' ')) - else: - # New. - flags.extend((yellow, '-', flag, '%', ' ')) - else: - for flag in sorted(enabled): - flags.extend((red, flag, ' ')) - for flag in sorted(disabled): - flags.extend((yellow, '-', flag, ' ')) - - # Only write this if we have something to write - if flags: - out.write(attr.upper(), '="') - # Omit the final space. - out.write(*flags[:-1]) - out.write('" ') - - def end(self): - if self.display_repo: - self.out.write() - repos = self.repos.items() - repos.sort(key=operator.itemgetter(1)) - for k, v in repos: - self.out.write(self.out.fg('blue'), "[%d] %s" % (v, k)) - - -class PaludisFormatter(Formatter): - - """Paludis formatter - - A Formatter designed to resemble Paludis' output - as much as much as possible. - """ - - def __init__(self, **kwargs): - Formatter.__init__(self, **kwargs) - self.packages = self.new = self.upgrades = self.downgrades = 0 - self.nslots = 0 - - def format(self, op): - out = self.out - origautoline = out.autoline - out.autoline = False - self.packages += 1 - - out.write('* ') - out.write(out.fg('blue'), op.pkg.key) - out.write("-%s" % op.pkg.fullver) - out.write("::%s " % op.pkg.repo.repo_id) - out.write(out.fg('blue'), "{:%s} " % op.pkg.slot) - if op.desc == 'add': - if op.pkg.slot != '0': - suffix = 'S' - self.nslots += 1 - else: - suffix = 'N' - self.new += 1 - out.write(out.fg('yellow'), "[%s]" % suffix) - elif op.desc == 'replace': - if op.pkg != op.old_pkg: - if op.pkg > op.old_pkg: - suffix = "U" - self.upgrades += 1 - else: - suffix = "D" - self.downgrades += 1 - out.write(out.fg('yellow'), "[%s %s]" % ( - suffix, op.old_pkg.fullver)) - else: - out.write(out.fg('yellow'), "[R]") - - red = out.fg('red') - green = out.fg('green') - flags = [] - use = set(op.pkg.use) - for flag in sorted(op.pkg.iuse): - if flag in use: - flags.extend((green, flag, ' ')) - else: - flags.extend((red, '-', flag, ' ')) - if flags: - out.write(' ') - # Throw away the final space. - out.write(*flags[:-1]) - out.write('\n') - out.autoline = origautoline - - def end(self): - self.out.write( - 'Total: %d packages ' - '(%d new, %d upgrades, %d downgrades, %d in new slots)' % ( - self.packages, self.new, self.upgrades, self.downgrades, - self.nslots)) - - -def formatter_factory_generator(cls): - """Factory for formatter factories that take no further arguments. - - A formatter factory is a subclass of Formatter or a callable - taking the same keyword arguments. - - This helper wraps such a subclass in an extra no-argument callable - that is usable by the configuration system. - """ - @configurable(typename='pmerge_formatter') - def factory(): - return cls - return factory - - -basic_factory = formatter_factory_generator(BasicFormatter) -pkgcore_factory = formatter_factory_generator(PkgcoreFormatter) -portage_factory = formatter_factory_generator(PortageFormatter) -paludis_factory = formatter_factory_generator(PaludisFormatter) - -@configurable(typename='pmerge_formatter') -def portage_verbose_factory(): - """Version of portage-formatter that is always in verbose mode.""" - def factory(**kwargs): - kwargs['display_repo'] = True - return PortageFormatter(**kwargs) - return factory diff --git a/pkgcore/ebuild/misc.py b/pkgcore/ebuild/misc.py deleted file mode 100644 index c1e17b3..0000000 --- a/pkgcore/ebuild/misc.py +++ /dev/null @@ -1,131 +0,0 @@ -# Copyright: 2005-2007 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -""" -misc. stuff we've not found a spot for yet. -""" - -from pkgcore.restrictions import packages, restriction -from pkgcore.ebuild.atom import atom -from pkgcore.ebuild.profiles import incremental_expansion - -from snakeoil.lists import iflatten_instance -from snakeoil.klass import generic_equality - -class collapsed_restrict_to_data(object): - - __metaclass__ = generic_equality - __attr_comparison__ = ('defaults', 'freeform', 'atoms', '__class__') - - def __init__(self, *restrict_sources): - """ - descriptive, no? - - Basically splits an iterable of restrict:data into - level of specificity, repo, cat, pkg, atom (dict) for use - in filters - """ - - always = [] - repo = [] - cat = [] - pkg = [] - atom_d = {} - for restrict_pairs in restrict_sources: - for a, data in restrict_pairs: - if not data: - continue - if isinstance(a, restriction.AlwaysBool): - # yes, odd attr name, but negate holds the val to return. - # note also, we're dropping AlwaysFalse; it'll never match. - if a.negate: - always.extend(data) - elif isinstance(a, atom): - atom_d.setdefault(a.key, []).append((a, data)) - elif isinstance(a, packages.PackageRestriction): - if a.attr == "category": - cat.append((a, data)) - elif a.attr == "package": - pkg.append((a, data)) - else: - raise ValueError("%r doesn't operate on package/category: " - "data %r" % (a, data)) - elif isinstance(a, restriction.AlwaysBool): - repo.append((a, data)) - else: - raise ValueError("%r is not a AlwaysBool, PackageRestriction, " - "or atom: data %r" % (a, data)) - - if always: - s = set() - incremental_expansion(s, always) - always = s - else: - always = set() - self.defaults = always - self.freeform = tuple(x for x in (repo, cat, pkg) if x) - self.atoms = atom_d - - def atom_intersects(self, atom): - return atom.key in self.atoms - - def pull_data(self, pkg, force_copy=False): - l = [] - for specific in self.freeform: - for restrict, data in specific: - if restrict.match(pkg): - l.append(data) - for atom, data in self.atoms.get(pkg.key, ()): - if atom.match(pkg): - l.append(data) - if not l: - if force_copy: - return set(self.defaults) - return self.defaults - s = set(self.defaults) - incremental_expansion(s, iflatten_instance(l)) - return s - - def iter_pull_data(self, pkg): - for item in self.defaults: - yield item - for specific in self.freeform: - for restrict, data in specific: - if restrict.match(pkg): - for item in data: - yield item - for atom, data in self.atoms.get(pkg.key, ()): - if atom.match(pkg): - for item in data: - yield item - - -class non_incremental_collapsed_restrict_to_data(collapsed_restrict_to_data): - - def pull_data(self, pkg, force_copy=False): - l = [] - for specific in self.freeform: - for restrict, data in specific: - if restrict.match(pkg): - l.append(data) - for atom, data in self.atoms.get(pkg.key, ()): - if atom.match(pkg): - l.append(data) - if not l: - if force_copy: - return set(self.defaults) - return self.defaults - s = set(self.defaults) - s.update(iflatten_instance(l)) - return s - - def iter_pull_data(self, pkg): - l = [self.defaults] - for specific in self.freeform: - l.extend(data for restrict, data in specific if restrict.match(pkg)) - for atom, data in self.atoms.get(pkg.key, ()): - if atom.match(pkg): - l.append(data) - if len(l) == 1: - return iter(self.defaults) - return iflatten_instance(l) diff --git a/pkgcore/ebuild/overlay_repository.py b/pkgcore/ebuild/overlay_repository.py deleted file mode 100644 index c19f0c2..0000000 --- a/pkgcore/ebuild/overlay_repository.py +++ /dev/null @@ -1,89 +0,0 @@ -# Copyright: 2006 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -""" -implementation of the standard PORTDIR + PORTDIR_OVERLAY repository stacking -""" - -from pkgcore.repository import prototype -from pkgcore.config import ConfigHint, errors -from pkgcore.ebuild import repository - -from snakeoil.lists import unstable_unique -from snakeoil.compatibility import all -from itertools import chain - -class OverlayRepo(prototype.tree): - - """ - Collapse multiple trees into one. - - Eclass dir is shared, the first package leftmost returned. - """ - - pkgcore_config_type = ConfigHint({'trees': 'refs:repo'}, typename='repo') - - configured = False - configurables = ("domain", "settings",) - configure = repository.ConfiguredTree - - # sucks a bit, need to work something better out here - format_magic = "ebuild_src" - - def __init__(self, trees, **kwds): - """ - @param trees: L{pkgcore.ebuild.repository.UnconfiguredTree} instances - to combine. - """ - - if not trees or len(trees) < 2: - raise errors.InstantiationError( - "Must specify at least two pathes to ebuild trees to overlay") - - self.trees = tuple(trees) - self._rv_trees = tuple(reversed(trees)) - self._version_owners = {} - prototype.tree.__init__(self) - - def _get_categories(self, category=None): - if category is not None: - updates = (tree.categories.get(category) for tree in self.trees) - updates = [x for x in updates if x is not None] - if not updates: - raise KeyError(category) - else: - updates = [tree.categories for tree in self.trees] - return tuple(set(chain(*updates))) - - def _get_packages(self, category): - updates = (tree.packages.get(category) for tree in self.trees) - updates = [x for x in updates if x is not None] - if not updates: - raise KeyError(category) - return tuple(set(chain(*updates))) - - def _get_versions(self, catpkg): - ver_owners = {} - fails = 0 - i = iter(self._rv_trees) - for tree in self._rv_trees: - new_vers = tree.versions.get(catpkg) - if new_vers is not None: - ver_owners.update((v, tree) for v in new_vers) - else: - fails += 1 - if fails == len(self._rv_trees): - raise KeyError(catpkg) - self._version_owners[catpkg] = tuple(ver_owners.iteritems()) - return tuple(ver_owners) - - def _internal_gen_candidates(self, candidates, sorter): - for cp in candidates: - if cp not in self.versions: - self.versions.get(cp) - for pkg in sorter(repo[cp + (ver,)] - for ver, repo in self._version_owners.get(cp, ())): - yield pkg - - def _visibility_limiters(self): - return [x for r in self.trees for x in r.default_visibility_limiters] diff --git a/pkgcore/ebuild/portage_conf.py b/pkgcore/ebuild/portage_conf.py deleted file mode 100644 index 37018ab..0000000 --- a/pkgcore/ebuild/portage_conf.py +++ /dev/null @@ -1,491 +0,0 @@ -# Copyright: 2006-2007 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -"""make.conf translator. - -Converts portage configuration files into L{pkgcore.config} form. -""" - -import os - -from pkgcore.config import basics, configurable -from pkgcore import const -from pkgcore.pkgsets.glsa import SecurityUpgrades - -from snakeoil.osutils import normpath, abspath, listdir_files, pjoin -from snakeoil.demandload import demandload -demandload(globals(), - 'errno', - 'pkgcore.config:errors', - 'pkgcore.log:logger', - 'ConfigParser:ConfigParser', - 'snakeoil.fileutils:read_bash_dict', - 'pkgcore.util:bzip2', - 'snakeoil.xml:etree', -) - - -def my_convert_hybrid(manager, val, arg_type): - """Modified convert_hybrid using a sequence of strings for section_refs.""" - if arg_type.startswith('refs:'): - subtype = 'ref:' + arg_type.split(':', 1)[1] - return list( - basics.LazyNamedSectionRef(manager, subtype, name) - for name in val) - return basics.convert_hybrid(manager, val, arg_type) - - -@configurable({'ebuild_repo': 'ref:repo', 'vdb': 'ref:repo', - 'profile': 'ref:profile'}, typename='pkgset') -def SecurityUpgradesViaProfile(ebuild_repo, vdb, profile): - """ - generate a GLSA vuln. pkgset limited by profile - - @param ebuild_repo: L{pkgcore.ebuild.repository.UnconfiguredTree} instance - @param vdb: L{pkgcore.repository.prototype.tree} instance that is the livefs - @param profile: L{pkgcore.ebuild.profiles} instance - """ - arch = profile.arch - if arch is None: - raise errors.InstantiationError("arch wasn't set in profiles") - return SecurityUpgrades(ebuild_repo, vdb, arch) - - -def add_layman_syncers(new_config, rsync_opts, overlay_paths, config_root='/', - default_loc="etc/layman/layman.cfg", - default_conf='overlays.xml'): - - try: - f = open(pjoin(config_root, default_loc)) - except IOError, ie: - if ie.errno != errno.ENOENT: - raise - return {} - - c = ConfigParser() - c.readfp(f) - storage_loc = c.get('MAIN', 'storage') - overlay_xml = pjoin(storage_loc, default_conf) - del c - - try: - xmlconf = etree.parse(overlay_xml) - except IOError, ie: - if ie.errno != errno.ENOENT: - raise - return {} - overlays = xmlconf.getroot() - if overlays.tag != 'overlays': - return {} - - new_syncers = {} - for overlay in overlays.findall('overlay'): - name = overlay.get('name') - src_type = overlay.get('type') - uri = overlay.get('src') - if None in (src_type, uri, name): - continue - path = pjoin(storage_loc, name) - if not os.path.exists(path): - continue - elif path not in overlay_paths: - continue - if src_type == 'tar': - continue - elif src_type == 'svn': - if uri.startswith('http://') or uri.startswith('https://'): - uri = 'svn+' + uri - elif src_type != 'rsync': - uri = '%s+%s' % (src_type, uri) - - new_syncers[path] = make_syncer(new_config, path, uri, rsync_opts, False) - return new_syncers - - -def isolate_rsync_opts(options): - """ - pop the misc RSYNC related options litered in make.conf, returning - a base rsync dict, and the full SYNC config - """ - base = {} - extra_opts = [] - - extra_opts.extend(options.pop('PORTAGE_RSYNC_EXTRA_OPTS', '').split()) - - ratelimit = options.pop('RSYNC_RATELIMIT', None) - if ratelimit is not None: - extra_opts.append('--bwlimit=%s' % ratelimit.strip()) - - # keep in mind this pops both potential vals. - retries = options.pop('PORTAGE_RSYNC_RETRIES', - options.pop('RSYNC_RETRIES', None)) - if retries is not None: - base['retries'] = retries.strip() - timeout = options.pop('RSYNC_TIMEOUT', None) - if timeout is not None: - base['timeout'] = timeout.strip() - - excludes = options.pop('RSYNC_EXCLUDEFROM', None) - if excludes is not None: - extra_opts.extend('--exclude-from=%s' % x - for x in excludes.split()) - - if extra_opts: - base['extra_opts'] = tuple(extra_opts) - - return base - - -def make_syncer(new_config, basedir, sync_uri, rsync_opts, - allow_timestamps=True): - d = {'basedir': basedir, 'uri': sync_uri} - if sync_uri.startswith('rsync'): - d.update(rsync_opts) - if allow_timestamps: - d['class'] = 'pkgcore.sync.rsync.rsync_timestamp_syncer' - else: - d['class'] = 'pkgcore.sync.rsync.rsync_syncer' - else: - d['class'] = 'pkgcore.sync.base.GenericSyncer' - - name = '%s syncer' % basedir - new_config[name] = basics.AutoConfigSection(d) - return name - - -def add_sets(config, root, portage_base_dir): - config["world"] = basics.AutoConfigSection({ - "class": "pkgcore.pkgsets.filelist.WorldFile", - "location": pjoin(root, const.WORLD_FILE)}) - config["system"] = basics.AutoConfigSection({ - "class": "pkgcore.pkgsets.system.SystemSet", - "profile": "profile"}) - config["installed"] = basics.AutoConfigSection({ - "class": "pkgcore.pkgsets.installed.Installed", - "vdb": "vdb"}) - config["versioned-installed"] = basics.AutoConfigSection({ - "class": "pkgcore.pkgsets.installed.VersionedInstalled", - "vdb": "vdb"}) - - set_fp = pjoin(portage_base_dir, "sets") - try: - for setname in listdir_files(set_fp): - # Potential for name clashes here, those will just make - # the set not show up in config. - if setname in ("system", "world"): - logger.warn("user defined set %s is disallowed; ignoring" % - pjoin(set_fp, setname)) - continue - config[setname] = basics.AutoConfigSection({ - "class":"pkgcore.pkgsets.filelist.FileList", - "location":pjoin(set_fp, setname)}) - except OSError, e: - if e.errno != errno.ENOENT: - raise - - -def add_profile(config, base_path): - make_profile = pjoin(base_path, 'make.profile') - try: - profile = normpath(abspath(pjoin( - base_path, os.readlink(make_profile)))) - except OSError, oe: - if oe.errno in (errno.ENOENT, errno.EINVAL): - raise errors.InstantiationError( - "%s must be a symlink pointing to a real target" % ( - make_profile,)) - raise errors.InstantiationError( - "%s: unexepect error- %s" % (make_profile, oe.strerror)) - - psplit = list(piece for piece in profile.split(os.path.sep) if piece) - # poor mans rindex. - try: - profile_start = psplit.index('profiles') - except ValueError: - raise errors.InstantiationError( - '%s expands to %s, but no profile detected' % ( - pjoin(base_path, 'make.profile'), profile)) - - config["profile"] = basics.AutoConfigSection({ - "class": "pkgcore.ebuild.profiles.OnDiskProfile", - "basepath": pjoin("/", *psplit[:profile_start + 1]), - "profile": pjoin(*psplit[profile_start + 1:])}) - - -def add_fetcher(config, conf_dict, distdir): - fetchcommand = conf_dict.pop("FETCHCOMMAND") - resumecommand = conf_dict.pop("RESUMECOMMAND", fetchcommand) - - # copy it to prevent modification. - fetcher_dict = dict(conf_dict) - # map a config arg to an obj arg, pop a few values - if "FETCH_ATTEMPTS" in fetcher_dict: - fetcher_dict["attempts"] = fetcher_dict.pop("FETCH_ATTEMPTS") - fetcher_dict.pop("readonly", None) - fetcher_dict.update( - {"class": "pkgcore.fetch.custom.fetcher", - "distdir": distdir, - "command": fetchcommand, - "resume_command": resumecommand - }) - config["fetcher"] = basics.AutoConfigSection(fetcher_dict) - - - -@configurable({'location': 'str'}, typename='configsection') -def config_from_make_conf(location="/etc/"): - """ - generate a config from a file location - - @param location: location the portage configuration is based in, - defaults to /etc - """ - - # this actually differs from portage parsing- we allow - # make.globals to provide vars used in make.conf, portage keeps - # them seperate (kind of annoying) - - config_root = os.environ.get("CONFIG_ROOT", "/") - base_path = pjoin(config_root, location.strip("/")) - portage_base = pjoin(base_path, "portage") - - # this isn't preserving incremental behaviour for features/use - # unfortunately - conf_dict = read_bash_dict(pjoin(base_path, "make.globals")) - conf_dict.update(read_bash_dict( - pjoin(base_path, "make.conf"), vars_dict=conf_dict, - sourcing_command="source")) - conf_dict.setdefault("PORTDIR", "/usr/portage") - root = os.environ.get("ROOT", conf_dict.get("ROOT", "/")) - gentoo_mirrors = list( - x+"/distfiles" for x in conf_dict.pop("GENTOO_MIRRORS", "").split()) - if not gentoo_mirrors: - gentoo_mirrors = None - - features = conf_dict.get("FEATURES", "").split() - - new_config = {} - - # sets... - add_sets(new_config, root, portage_base) - add_profile(new_config, base_path) - - kwds = {"class": "pkgcore.vdb.repository", - "location": pjoin(root, 'var', 'db', 'pkg')} - kwds["cache_location"] = pjoin(config_root, 'var', 'cache', 'edb', - 'dep', 'var', 'db', 'pkg') - new_config["vdb"] = basics.AutoConfigSection(kwds) - - portdir = normpath(conf_dict.pop("PORTDIR").strip()) - portdir_overlays = [ - normpath(x) for x in conf_dict.pop("PORTDIR_OVERLAY", "").split()] - - - # define the eclasses now. - all_ecs = [] - for x in [portdir] + portdir_overlays: - ec_path = pjoin(x, "eclass") - new_config[ec_path] = basics.AutoConfigSection({ - "class": "pkgcore.ebuild.eclass_cache.cache", - "path": ec_path, - "portdir": portdir}) - all_ecs.append(ec_path) - - new_config['ebuild-repo-common'] = basics.AutoConfigSection({ - 'class': 'pkgcore.ebuild.repository.tree', - 'default_mirrors': gentoo_mirrors, - 'inherit-only': True, - 'eclass_cache': 'eclass stack'}) - new_config['cache-common'] = basics.AutoConfigSection({ - 'class': 'pkgcore.cache.flat_hash.database', - 'inherit-only': True, - 'location': pjoin(config_root, 'var', 'cache', 'edb', 'dep'), - }) - - - # used by PORTDIR syncer, and any layman defined syncers - rsync_opts = isolate_rsync_opts(conf_dict) - portdir_syncer = conf_dict.pop("SYNC", None) - - if portdir_overlays and '-layman-sync' not in features: - overlay_syncers = add_layman_syncers(new_config, rsync_opts, - portdir_overlays, config_root=config_root) - else: - overlay_syncers = {} - - for tree_loc in portdir_overlays: - kwds = { - 'inherit': ('ebuild-repo-common',), - 'location': tree_loc, - 'cache': (basics.AutoConfigSection({ - 'inherit': ('cache-common',), - 'label': tree_loc}),), - 'class': 'pkgcore.ebuild.repository.SlavedTree', - 'parent_repo': 'portdir' - } - if tree_loc in overlay_syncers: - kwds['sync'] = overlay_syncers[tree_loc] - new_config[tree_loc] = basics.AutoConfigSection(kwds) - - rsync_portdir_cache = os.path.exists(pjoin(portdir, "metadata", "cache")) \ - and "metadata-transfer" not in features - - # if a metadata cache exists, use it - if rsync_portdir_cache: - new_config["portdir cache"] = basics.AutoConfigSection({ - 'class': 'pkgcore.cache.metadata.database', - 'location': portdir, - 'label': 'portdir cache', - 'readonly': 'yes'}) - else: - new_config["portdir cache"] = basics.AutoConfigSection({ - 'inherit': ('cache-common',), - 'label': portdir}) - - base_portdir_config = {} - if portdir_syncer is not None: - base_portdir_config = {"sync": make_syncer(new_config, portdir, - portdir_syncer, rsync_opts)} - - # setup portdir. - cache = ('portdir cache',) - if not portdir_overlays: - d = dict(base_portdir_config) - d['inherit'] = ('ebuild-repo-common',) - d['location'] = portdir - d['cache'] = ('portdir cache',) - - new_config[portdir] = basics.FakeIncrementalDictConfigSection( - my_convert_hybrid, d) - new_config["eclass stack"] = basics.section_alias( - pjoin(portdir, 'eclass'), 'eclass_cache') - new_config['portdir'] = basics.section_alias(portdir, 'repo') - new_config['repo-stack'] = basics.section_alias(portdir, 'repo') - else: - # There's always at least one (portdir) so this means len(all_ecs) > 1 - new_config['%s cache' % (portdir,)] = basics.AutoConfigSection({ - 'inherit': ('cache-common',), - 'label': portdir}) - cache = ('portdir cache',) - if rsync_portdir_cache: - cache = ('%s cache' % (portdir,),) + cache - - d = dict(base_portdir_config) - d['inherit'] = ('ebuild-repo-common',) - d['location'] = portdir - d['cache'] = cache - - new_config[portdir] = basics.FakeIncrementalDictConfigSection( - my_convert_hybrid, d) - - if rsync_portdir_cache: - # created higher up; two caches, writes to the local, - # reads (when possible) from pregenned metadata - cache = ('portdir cache',) - else: - cache = ('%s cache' % (portdir,),) - new_config['portdir'] = basics.FakeIncrementalDictConfigSection( - my_convert_hybrid, { - 'inherit': ('ebuild-repo-common',), - 'location': portdir, - 'cache': cache, - 'eclass_cache': pjoin(portdir, 'eclass')}) - - # reverse the ordering so that overlays override portdir - # (portage default) - new_config["eclass stack"] = basics.FakeIncrementalDictConfigSection( - my_convert_hybrid, { - 'class': 'pkgcore.ebuild.eclass_cache.StackedCaches', - 'eclassdir': pjoin(portdir, "eclass"), - 'caches': tuple(reversed(all_ecs))}) - - new_config['repo-stack'] = basics.FakeIncrementalDictConfigSection( - my_convert_hybrid, { - 'class': 'pkgcore.ebuild.overlay_repository.OverlayRepo', - 'trees': tuple(reversed([portdir] + portdir_overlays))}) - - # disabled code for using portage config defined cache modules; - # need to re-examine and see if they're still in sync with our cache subsystem -# if os.path.exists(base_path+"portage/modules"): -# pcache = read_dict( -# base_path+"portage/modules").get("portdbapi.auxdbmodule", None) - -# cache_config = {"type": "cache", -# "location": "%s/var/cache/edb/dep" % -# config_root.rstrip("/"), -# "label": "make_conf_overlay_cache"} -# if pcache is None: -# if portdir_overlays or ("metadata-transfer" not in features): -# cache_config["class"] = "pkgcore.cache.flat_hash.database" -# else: -# cache_config["class"] = "pkgcore.cache.metadata.database" -# cache_config["location"] = portdir -# cache_config["readonly"] = "true" -# else: -# cache_config["class"] = pcache -# -# new_config["cache"] = basics.ConfigSectionFromStringDict( -# "cache", cache_config) - - - new_config['vuln'] = basics.AutoConfigSection({ - 'class': SecurityUpgradesViaProfile, - 'ebuild_repo': 'repo-stack', - 'vdb': 'vdb', - 'profile': 'profile'}) - new_config['glsa'] = basics.section_alias('vuln', - SecurityUpgradesViaProfile.pkgcore_config_type.typename) - #binpkg. - pkgdir = conf_dict.pop('PKGDIR', None) - default_repos = ('repo-stack',) - if pkgdir is not None: - try: - pkgdir = abspath(pkgdir) - except OSError, oe: - if oe.errno != errno.ENOENT: - raise - pkgdir = None - # If we are not using the native bzip2 then the Tarfile.bz2open - # the binpkg repository uses will fail. - if pkgdir and os.path.isdir(pkgdir): - if not bzip2.native: - logger.warn("python's bz2 module isn't available: " - "disabling binpkg support") - else: - new_config['binpkg'] = basics.ConfigSectionFromStringDict({ - 'class': 'pkgcore.binpkg.repository.tree', - 'location': pkgdir}) - default_repos += ('binpkg',) - - # now add the fetcher- we delay it till here to clean out the environ - # it passes to the command. - # *everything* in the conf_dict must be str values also. - distdir = normpath(conf_dict.pop("DISTDIR", pjoin(portdir, "distdir"))) - add_fetcher(new_config, conf_dict, distdir) - - # finally... domain. - conf_dict.update({ - 'class': 'pkgcore.ebuild.domain.domain', - 'repositories': default_repos, - 'fetcher': 'fetcher', - 'default': True, - 'vdb': ('vdb',), - 'profile': 'profile', - 'name': 'livefs domain', - 'root':root}) - for f in ( - "package.mask", "package.unmask", "package.keywords", "package.use", - "bashrc"): - fp = pjoin(portage_base, f) - try: - os.stat(fp) - except OSError, oe: - if oe.errno != errno.ENOENT: - raise - else: - conf_dict[f] = fp - - new_config['livefs domain'] = basics.FakeIncrementalDictConfigSection( - my_convert_hybrid, conf_dict) - - return new_config diff --git a/pkgcore/ebuild/processor.py b/pkgcore/ebuild/processor.py deleted file mode 100644 index 047d00e..0000000 --- a/pkgcore/ebuild/processor.py +++ /dev/null @@ -1,703 +0,0 @@ -# Copyright: 2004-2006 Brian Harring <ferringb@gmail.com> -# License: GPL2 - - -""" -low level ebuild processor. - -This basically is a coprocessor that controls a bash daemon for actual -ebuild execution. Via this, the bash side can reach into the python -side (and vice versa), enabling remote trees (piping data from python -side into bash side for example). - -A couple of processors are left lingering while pkgcore is running for -the purpose of avoiding spawning overhead, this (and the general -design) reduces regen time by over 40% compared to portage-2.1 -""" - -# this needs work. it's been pruned heavily from what ebd used -# originally, but it still isn't what I would define as 'right' - - -__all__ = ( - "request_ebuild_processor", "release_ebuild_processor", "EbuildProcessor" - "UnhandledCommand", "expected_ebuild_env") - - -inactive_ebp_list = [] -active_ebp_list = [] - -import pkgcore.spawn, os, signal, errno, sys -from pkgcore.const import ( - depends_phase_path, EBUILD_DAEMON_PATH, EBUILD_ENV_PATH, EBD_ENV_PATH) -from pkgcore.os_data import portage_uid, portage_gid - -from snakeoil.currying import post_curry, partial -from snakeoil.demandload import demandload -demandload(globals(), - 'pkgcore.log:logger', - 'snakeoil:osutils', -) - -import traceback - -def shutdown_all_processors(): - """kill off all known processors""" - try: - while active_ebp_list: - try: - active_ebp_list.pop().shutdown_processor( - ignore_keyboard_interrupt=True) - except (IOError, OSError): - pass - - while inactive_ebp_list: - try: - inactive_ebp_list.pop().shutdown_processor( - ignore_keyboard_interrupt=True) - except (IOError, OSError): - pass - except Exception,e: - traceback.print_exc() - print e - raise - -pkgcore.spawn.atexit_register(shutdown_all_processors) - -def request_ebuild_processor(userpriv=False, sandbox=None, fakeroot=False, - save_file=None): - """ - request an ebuild_processor instance, creating a new one if needed. - - Note that fakeroot processes are B{never} reused due to the fact - the fakeroot env becomes localized to the pkg it's handling. - - @return: L{EbuildProcessor} - @param userpriv: should the processor be deprived to - L{pkgcore.os_data.portage_gid} and L{pkgcore.os_data.portage_uid}? - @param sandbox: should the processor be sandboxed? - @param fakeroot: should the processor be fakerooted? This option is - mutually exclusive to sandbox, and requires save_file to be set. - @param save_file: location to store fakeroot state dumps - """ - - if sandbox is None: - sandbox = pkgcore.spawn.is_sandbox_capable() - - if not fakeroot: - for x in inactive_ebp_list: - if x.userprived() == userpriv and (x.sandboxed() or not sandbox): - if not x.is_alive: - inactive_ebp_list.remove(x) - continue - inactive_ebp_list.remove(x) - active_ebp_list.append(x) - return x - - e = EbuildProcessor(userpriv, sandbox, fakeroot, save_file) - active_ebp_list.append(e) - return e - - -def release_ebuild_processor(ebp): - """ - the inverse of request_ebuild_processor. - - Any processor requested via request_ebuild_processor B{must} be released - via this function once it's no longer in use. - This includes fakerooted processors. - - @param ebp: L{EbuildProcessor} instance - @return: boolean indicating release results- if the processor isn't known - as active, False is returned. - If a processor isn't known as active, this means either calling - error or an internal error. - """ - - try: - active_ebp_list.remove(ebp) - except ValueError: - return False - - assert ebp not in inactive_ebp_list - # if it's a fakeroot'd process, we throw it away. - # it's not useful outside of a chain of calls - if ebp.onetime() or ebp.locked: - # ok, so the thing is not reusable either way. - ebp.shutdown_processor() - else: - inactive_ebp_list.append(ebp) - return True - - -class ProcessingInterruption(Exception): - pass - -class FinishedProcessing(ProcessingInterruption): - - def __init__(self, val, msg=None): - ProcessingInterruption.__init__( - self, "Finished processing with val, %s" % (val,)) - self.val, self.msg = val, msg - -class UnhandledCommand(ProcessingInterruption): - - def __init__(self, line=None): - ProcessingInterruption.__init__( - self, "unhandled command, %s" % (line,)) - self.line = line - -def chuck_KeyboardInterrupt(*arg): - raise KeyboardInterrupt("ctrl+c encountered") - -def chuck_UnhandledCommand(processor, line): - raise UnhandledCommand(line) - -def chuck_StoppingCommand(val, processor, *args): - if callable(val): - raise FinishedProcessing(val(args[0])) - raise FinishedProcessing(val) - - -class InitializationError(Exception): - pass - - - - -class EbuildProcessor(object): - - """abstraction of a running ebuild.sh instance. - - Contains the env, functions, etc that ebuilds expect. - """ - - def __init__(self, userpriv, sandbox, fakeroot, save_file): - """ - @param sandbox: enables a sandboxed processor - @param userpriv: enables a userpriv'd processor - @param fakeroot: enables a fakeroot'd processor- - this is a mutually exclusive option to sandbox, and - requires userpriv to be enabled. Violating this will - result in nastyness. - """ - - self.lock() - self.ebd = EBUILD_DAEMON_PATH - spawn_opts = {} - - if fakeroot and (sandbox or not userpriv): - traceback.print_stack() - print "warning, was asking to enable fakeroot but-" - print "sandbox", sandbox, "userpriv", userpriv - print "this isn't valid. bailing" - raise InitializationError("cannot initialize with sandbox and fakeroot") - - if userpriv: - self.__userpriv = True - spawn_opts.update({ - "uid":portage_uid, "gid":portage_gid, - "groups":[portage_gid], "umask":002}) - else: - if pkgcore.spawn.is_userpriv_capable(): - spawn_opts.update({"gid":portage_gid, - "groups":[0, portage_gid]}) - self.__userpriv = False - - # open the pipes to be used for chatting with the new daemon - cread, cwrite = os.pipe() - dread, dwrite = os.pipe() - self.__sandbox = False - self.__fakeroot = False - - # since it's questionable which spawn method we'll use (if - # sandbox or fakeroot fex), we ensure the bashrc is invalid. - env = dict((x, "/etc/portage/spork/not/valid/ha/ha") - for x in ("BASHRC", "BASH_ENV")) - args = [] - if sandbox: - if not pkgcore.spawn.is_sandbox_capable(): - raise ValueError("spawn lacks sandbox capabilities") - if fakeroot: - raise InitializationError('fakeroot was on, but sandbox was also on') - self.__sandbox = True - spawn_func = pkgcore.spawn.spawn_sandbox -# env.update({"SANDBOX_DEBUG":"1", "SANDBOX_DEBUG_LOG":"/var/tmp/test"}) - - elif fakeroot: - if not pkgcore.spawn.is_fakeroot_capable(): - raise ValueError("spawn lacks fakeroot capabilities") - self.__fakeroot = True - spawn_func = pkgcore.spawn.spawn_fakeroot - args.append(save_file) - else: - spawn_func = pkgcore.spawn.spawn - - # force to a neutral dir so that sandbox/fakeroot won't explode if - # ran from a nonexistant dir - spawn_opts["chdir"] = "/tmp" - # little trick. we force the pipes to be high up fd wise so - # nobody stupidly hits 'em. - max_fd = min(pkgcore.spawn.max_fd_limit, 1024) - env.update({ - "EBD_READ_FD": str(max_fd -2), "EBD_WRITE_FD": str(max_fd -1)}) - self.pid = spawn_func("/bin/bash %s daemonize" % self.ebd, \ - fd_pipes={0:0, 1:1, 2:2, max_fd-2:cread, max_fd-1:dwrite}, \ - returnpid=True, env=env, *args, **spawn_opts)[0] - - os.close(cread) - os.close(dwrite) - self.ebd_write = os.fdopen(cwrite, "w") - self.ebd_read = os.fdopen(dread, "r") - - # basically a quick "yo" to the daemon - self.write("dude?") - if not self.expect("dude!"): - print "error in server coms, bailing." - raise InitializationError( - "expected 'dude!' response from ebd, which wasn't received. " - "likely a bug") - self.write(EBD_ENV_PATH) - self.write(sys.executable) - self.write(osutils.normpath(osutils.abspath(osutils.join( - pkgcore.__file__, os.pardir, os.pardir)))) - if self.__sandbox: - self.write("sandbox_log?") - self.__sandbox_log = self.read().split()[0] - self.dont_export_vars = self.read().split() - # locking isn't used much, but w/ threading this will matter - self.unlock() - - - def prep_phase(self, phase, env, sandbox=None, logging=None): - """ - Utility function, to initialize the processor for a phase. - - Used to combine multiple calls into one, leaving the processor - in a state where all that remains is a call start_processing - call, then generic_handler event loop. - - @param phase: phase to prep for - @type phase: str - @param env: mapping of the environment to prep the processor with - @param sandbox: should the sandbox be enabled? - @param logging: None, or a filepath to log the output from the - processor to - @return: True for success, False for everything else - """ - - self.write("process_ebuild %s" % phase) - if not self.send_env(env): - return False - if sandbox: - self.set_sandbox_state(sandbox) - if logging: - if not self.set_logfile(logging): - return False - return True - - def sandboxed(self): - """is this instance sandboxed?""" - return self.__sandbox - - def userprived(self): - """is this instance userprived?""" - return self.__userpriv - - def fakerooted(self): - """is this instance fakerooted?""" - return self.__fakeroot - - def onetime(self): - """Is this instance going to be discarded after usage (fakerooted)?""" - return self.__fakeroot - - def write(self, string, flush=True, disable_runtime_exceptions=False): - """send something to the bash side. - - @param string: string to write to the bash processor. - All strings written are automatically \\n terminated. - @param flush: boolean controlling whether the data is flushed - immediately. Disabling flush is useful when dumping large - amounts of data. - """ - string = str(string) - try: - if string == "\n": - self.ebd_write.write(string) - else: - self.ebd_write.write(string +"\n") - if flush: - self.ebd_write.flush() - except IOError, ie: - if ie.errno == errno.EPIPE and not disable_runtime_exceptions: - raise RuntimeError(ie) - raise - - def expect(self, want): - """read from the daemon, check if the returned string is expected. - - @param want: string we're expecting - @return: boolean, was what was read == want? - """ - got = self.read() - return want == got.rstrip("\n") - - def read(self, lines=1, ignore_killed=False): - """ - read data from the daemon. Shouldn't be called except internally - """ - mydata = [] - while lines > 0: - mydata.append(self.ebd_read.readline()) - if mydata[-1].startswith("killed"): -# self.shutdown_processor() - chuck_KeyboardInterrupt() - lines -= 1 - return "\n".join(mydata) - - def sandbox_summary(self, move_log=False): - """ - if the instance is sandboxed, print the sandbox access summary - - @param move_log: location to move the sandbox log to if a failure - occured - """ - if not os.path.exists(self.__sandbox_log): - self.write("end_sandbox_summary") - return 0 - violations = [x.strip() - for x in open(self.__sandbox_log, "r") if x.strip()] - if not violations: - self.write("end_sandbox_summary") - return 0 - if not move_log: - move_log = self.__sandbox_log - elif move_log != self.__sandbox_log: - myf = open(move_log) - for x in violations: - myf.write(x+"\n") - myf.close() - # XXX this is fugly, use a colorizer or something - # (but it is better than "from output import red" (portage's output)) - def red(text): - return '\x1b[31;1m%s\x1b[39;49;00m' % (text,) - self.ebd_write.write(red( - "--------------------------- ACCESS VIOLATION SUMMARY " - "---------------------------")+"\n") - self.ebd_write.write(red("LOG FILE = \"%s\"" % move_log)+"\n\n") - for x in violations: - self.ebd_write.write(x+"\n") - self.write(red( - "-----------------------------------------------------" - "---------------------------")+"\n") - self.write("end_sandbox_summary") - try: - os.remove(self.__sandbox_log) - except (IOError, OSError), e: - print "exception caught when cleansing sandbox_log=%s" % str(e) - return 1 - - def preload_eclasses(self, ec_file): - """ - Preload an eclass into a bash function. - - Avoids the cost of going to disk on inherit. Preloading eutils - (which is heaviliy inherited) speeds up regen times for - example. - - @param ec_file: filepath of eclass to preload - @return: boolean, True for success - """ - if not os.path.exists(ec_file): - return 1 - self.write("preload_eclass %s" % ec_file) - if self.expect("preload_eclass succeeded"): - self.preloaded_eclasses = True - return True - return False - - def lock(self): - """ - lock the processor. Currently doesn't block any access, but will - """ - self.processing_lock = True - - def unlock(self): - """ - unlock the processor - """ - self.processing_lock = False - - @property - def locked(self): - """ - is the processor locked? - """ - return self.processing_lock - - @property - def is_alive(self): - """ - returns if it's known if the processor has been shutdown. - - Currently doesn't check to ensure the pid is still running, - yet it should. - """ - try: - if self.pid is None: - return False - try: - os.kill(self.pid, 0) - return True - except OSError: - # pid is dead. - pass - self.pid = None - return False - - except AttributeError: - # thrown only if failure occured instantiation. - return False - - def shutdown_processor(self, ignore_keyboard_interrupt=False): - """ - tell the daemon to shut itself down, and mark this instance as dead - """ - try: - if self.is_alive: - self.write("shutdown_daemon", disable_runtime_exceptions=True) - self.ebd_write.close() - self.ebd_read.close() - else: - return - except (IOError, OSError, ValueError): - os.kill(self.pid, signal.SIGTERM) - - # now we wait. - try: - os.waitpid(self.pid, 0) - except KeyboardInterrupt: - if not ignore_keyboard_interrupt: - raise - - # currently, this assumes all went well. - # which isn't always true. - self.pid = None - - def set_sandbox_state(self, state): - """ - tell the daemon whether to enable the sandbox, or disable it - @param state: boolean, if True enable sandbox - """ - if state: - self.write("set_sandbox_state 1") - else: - self.write("set_sandbox_state 0") - - def send_env(self, env_dict): - """ - transfer the ebuild's desired env (env_dict) to the running daemon - - @type env_dict: mapping with string keys and values. - @param env_dict: the bash env. - """ - - self.write("start_receiving_env\n") - exported_keys = [] - data = [] - for x in env_dict: - if x not in self.dont_export_vars: - if not x[0].isalpha(): - raise KeyError(x) - s = env_dict[x].replace("\\", "\\\\\\\\") - s = s.replace("'", "\\\\'") - s = s.replace("\n", "\\\n") - data.append("%s=$'%s'\n" % (x, s)) - exported_keys.append(x) - if exported_keys: - data.append("export %s\n" % ' '.join(exported_keys)) - data.append("end_receiving_env") - self.write(''.join(data), flush=True) - return self.expect("env_received") - - def set_logfile(self, logfile=''): - """ - Set the logfile (location to log to). - - Relevant only when the daemon is sandbox'd, - - @param logfile: filepath to log to - """ - self.write("logging %s" % logfile) - return self.expect("logging_ack") - - def __del__(self): - """simply attempts to notify the daemon to die""" - # for this to be reached means we ain't in a list no more. - if self.is_alive: - # I'd love to know why the exception wrapping is required... - try: - self.shutdown_processor() - except TypeError: - pass - - def get_keys(self, package_inst, eclass_cache): - """ - request the metadata be regenerated from an ebuild - - @param package_inst: L{pkgcore.ebuild.ebuild_src.package} instance - to regenerate - @param eclass_cache: L{pkgcore.ebuild.eclass_cache} instance to use - for eclass access - @return: dict when successful, None when failed - """ - - self.write("process_ebuild depend") - e = expected_ebuild_env(package_inst) - e["PATH"] = depends_phase_path - self.send_env(e) - self.set_sandbox_state(True) - self.write("start_processing") - - metadata_keys = {} - val = self.generic_handler(additional_commands={ - "request_inherit": post_curry( - self.__class__._inherit, eclass_cache), - "key": post_curry(self.__class__._receive_key, metadata_keys)}) - - if not val: - logger.error("returned val from get_keys was '%s'" % str(val)) - raise Exception(val) - - return metadata_keys - - def _receive_key(self, line, keys_dict): - """ - internal function used for receiving keys from the bash processor - """ - line = line.split("=", 1) - if len(line) != 2: - raise FinishedProcessing(True) - else: - keys_dict[line[0]] = line[1] - - def _inherit(self, line, ecache): - """ - Callback for implementing inherit digging into eclass_cache. - - Not for normal consumption. - """ - if line is None: - self.write("failed") - raise UnhandledCommand( - "inherit requires an eclass specified, none specified") - - line = line.strip() - eclass = ecache.get_eclass(line) - if eclass is None: - self.write("failed") - raise UnhandledCommand( - "inherit requires an unknown eclass, %s cannot be found" % line) - - if eclass.get_path is not None: - value = eclass.get_path() - self.write("path") - self.write(value) - else: - # XXX $10 this doesn't work. - value = eclass.get_fileobj().read() - self.write("transfer") - self.write(value) - - # this basically handles all hijacks from the daemon, whether - # confcache or portageq. - def generic_handler(self, additional_commands=None): - """ - internal event handler responding to the running processor's requests. - - @type additional_commands: mapping from string to callable. - @param additional_commands: Extra command handlers. - Command names cannot have spaces. - The callable is called with the processor as first arg, and - remaining string (None if no remaining fragment) as second arg. - If you need to split the args to command, whitespace splitting - falls to your func. - - @raise UnhandledCommand: thrown when an unknown command is encountered. - """ - - # note that self is passed in. so... we just pass in the - # unbound instance. Specifically, via digging through - # __class__ if you don't do it, sandbox_summary (fex) cannot - # be overriden, this func will just use this classes version. - # so dig through self.__class__ for it. :P - - handlers = {"request_sandbox_summary":self.__class__.sandbox_summary} - f = chuck_UnhandledCommand - for x in ("prob", "env_receiving_failed", "failed"): - handlers[x] = f - del f - - handlers["phases"] = partial( - chuck_StoppingCommand, lambda f: f.lower().strip() == "succeeded") - - handlers["killed"] = chuck_KeyboardInterrupt - - if additional_commands is not None: - for x in additional_commands: - if not callable(additional_commands[x]): - raise TypeError(additional_commands[x]) - - handlers.update(additional_commands) - - self.lock() - - try: - while True: - line = self.read().strip() - # split on first whitespace. - s = line.split(None, 1) - if s[0] in handlers: - if len(s) == 1: - s.append(None) - handlers[s[0]](self, s[1]) - else: - logger.error("unhandled command '%s', line '%s'" % - (s[0], line)) - raise UnhandledCommand(line) - - except FinishedProcessing, fp: - v = fp.val - self.unlock() - return v - - -def expected_ebuild_env(pkg, d=None, env_source_override=None): - """ - setup expected ebuild vars - - @param d: if None, generates a dict, else modifies a passed in mapping - @return: mapping - """ - if d is None: - d = {} - d["CATEGORY"] = pkg.category - d["PF"] = "-".join((pkg.package, pkg.fullver)) - d["P"] = "-".join((pkg.package, pkg.version)) - d["PN"] = pkg.package - d["PV"] = pkg.version - if pkg.revision is None: - d["PR"] = "r0" - else: - d["PR"] = "r%i" % pkg.revision - d["PVR"] = pkg.fullver - if env_source_override: - path = env_source_override.get_path - if path is not None: - d["EBUILD"] = path() - else: - d["EBUILD"] = pkg.ebuild.get_path() - d["PATH"] = ":".join(EBUILD_ENV_PATH + d.get("PATH", "").split(":")) - return d - diff --git a/pkgcore/ebuild/profiles.py b/pkgcore/ebuild/profiles.py deleted file mode 100644 index 8ef19de..0000000 --- a/pkgcore/ebuild/profiles.py +++ /dev/null @@ -1,562 +0,0 @@ -# Copyright: 2006-2007 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -import errno, os -from itertools import chain - -from pkgcore.config import ConfigHint -from pkgcore.ebuild import const -from pkgcore.ebuild import ebuild_src -from pkgcore.repository import virtual - -from snakeoil.osutils import abspath, join as pjoin, readlines -from snakeoil.containers import InvertedContains -from snakeoil.fileutils import iter_read_bash, read_bash_dict -from snakeoil.caching import WeakInstMeta -from snakeoil.currying import partial -from snakeoil.demandload import demandload - -demandload(globals(), - 'pkgcore.interfaces.data_source:local_source', - 'pkgcore.ebuild:cpv', - 'pkgcore.ebuild:atom', - 'pkgcore.repository:util', - 'pkgcore.restrictions:packages', -) - -class ProfileError(Exception): - - def __init__(self, path, filename, error): - self.path, self.filename, self.error = path, filename, error - - def __str__(self): - return "ProfileError: profile %r, file %r, error %s" % ( - self.path, self.filename, self.error) - -def load_decorator(filename, handler=iter_read_bash, fallback=()): - def f(func): - def f2(self, *args): - path = pjoin(self.path, filename) - try: - data = readlines(path, False, True, True) - if data is None: - return func(self, fallback, *args) - return func(self, handler(data), *args) - except (KeyboardInterrupt, RuntimeError, SystemExit): - raise - except Exception, e: - raise ProfileError(self.path, filename, e) - return f2 - return f - -def split_negations(data, func): - neg, pos = [], [] - for line in data: - if line[0] == '-': - if len(line) == 1: - raise ValueError("'-' negation without a token") - neg.append(func(line[1:])) - else: - pos.append(func(line)) - return (tuple(neg), tuple(pos)) - - -class ProfileNode(object): - - __metaclass__ = WeakInstMeta - __inst_caching__ = True - - def __init__(self, path): - if not os.path.isdir(path): - raise ProfileError(path, "", "profile doesn't exist") - self.path = path - - def __str__(self): - return "Profile at %r" % self.path - - def __repr__(self): - return '<%s path=%r, @%#8x>' % (self.__class__.__name__, self.path, - id(self)) - - @load_decorator("packages") - def _load_packages(self, data): - # sys packages and visibility - sys, neg_sys, vis, neg_vis = [], [], [], [] - for line in data: - if line[0] == '-': - if line[1] == '*': - neg_sys.append(atom.atom(line[2:])) - else: - neg_vis.append(atom.atom(line[1:], negate_vers=True)) - else: - if line[0] == '*': - sys.append(atom.atom(line[1:])) - else: - vis.append(atom.atom(line, negate_vers=True)) - - self.system = (tuple(neg_sys), tuple(sys)) - self.visibility = (tuple(neg_vis), tuple(vis)) - - @load_decorator("parent") - def _load_parents(self, data): - self.parents = tuple(ProfileNode(abspath(pjoin(self.path, x))) - for x in data) - return self.parents - - @load_decorator("package.provided") - def _load_pkg_provided(self, data): - self.pkg_provided = split_negations(data, cpv.CPV) - return self.pkg_provided - - @load_decorator("virtuals") - def _load_virtuals(self, data): - d = {} - for line in data: - l = line.split() - if len(l) != 2: - raise ValueError("%r is malformated" % line) - d[cpv.CPV(l[0]).package] = atom.atom(l[1]) - self.virtuals = d - return d - - @load_decorator("package.mask") - def _load_masks(self, data): - self.masks = split_negations(data, atom.atom) - return self.masks - - @load_decorator("deprecated", lambda i:i, None) - def _load_deprecated(self, data): - if data is not None: - data = iter(data) - try: - replacement = data.next().strip() - msg = "\n".join(x.lstrip("#").strip() - for x in data) - data = (replacement, msg) - except StopIteration: - # only an empty replacement could trigger this; thus - # formatted badly. - raise ValueError("didn't specify a replacement profile") - self.deprecated = data - return data - - @load_decorator("use.mask") - def _load_masked_use(self, data): - d = self._load_pkg_use_mask() - neg, pos = split_negations(data, str) - if neg or pos: - d[packages.AlwaysTrue] = (neg, pos) - self.masked_use = d - return d - - @load_decorator("package.use.mask") - def _load_pkg_use_mask(self, data): - d = {} - for line in data: - i = iter(line.split()) - a = atom.atom(i.next()) - neg, pos = d.setdefault(a, ([], [])) - for x in i: - if x[0] == '-': - neg.append(x[1:]) - else: - pos.append(x) - for k, v in d.iteritems(): - d[k] = tuple(tuple(x) for x in v) - self.pkg_use_mask = d - return d - - @load_decorator("package.use") - def _load_pkg_use(self, data): - d = {} - for line in data: - i = iter(line.split()) - a = atom.atom(i.next()) - neg, pos = d.setdefault(a, ([], [])) - for x in i: - if x[0] == '-': - neg.append(x[1:]) - else: - pos.append(x) - for k, v in d.iteritems(): - d[k] = tuple(tuple(x) for x in v) - self.pkg_use = d - return d - - @load_decorator("use.force") - def _load_forced_use(self, data): - d = self._load_pkg_use_force() - neg, pos = split_negations(data, str) - if neg or pos: - d[packages.AlwaysTrue] = (neg, pos) - self.forced_use = d - return d - - @load_decorator("package.use.force") - def _load_pkg_use_force(self, data): - d = {} - for line in data: - i = iter(line.split()) - a = atom.atom(i.next()) - neg, pos = d.setdefault(a, ([], [])) - for x in i: - if x[0] == '-': - neg.append(x[1:]) - else: - pos.append(x) - for k, v in d.iteritems(): - d[k] = tuple(tuple(x) for x in v) - self.pkg_use_force = d - return d - - def _load_default_env(self): - path = pjoin(self.path, "make.defaults") - try: - f = open(path, "r") - except IOError, ie: - if ie.errno != errno.ENOENT: - raise ProfileError(self.path, "make.defaults", ie) - self.default_env = {} - return self.default_env - try: - try: - d = read_bash_dict(f) - finally: - f.close() - except (KeyboardInterrupt, RuntimeError, SystemExit): - raise - except Exception ,e: - raise ProfileError(self.path, "make.defaults", e) - self.default_env = d - return d - - def _load_bashrc(self): - path = pjoin(self.path, "profile.bashrc") - if os.path.exists(path): - self.bashrc = local_source(path) - else: - self.bashrc = None - return self.bashrc - - def __getattr__(self, attr): - if attr in ("system", "visibility"): - self._load_packages() - return getattr(self, attr) - # use objects getattr to bypass our own; prevents infinite recursion - # if they request something non existant - try: - func = object.__getattribute__(self, "_load_%s" % attr) - except AttributeError: - raise AttributeError(self, attr) - if func is None: - raise AttributeError(attr) - return func() - - -class EmptyRootNode(ProfileNode): - - __inst_caching__ = True - - parents = () - deprecated = None - forced_use = masked_use = {} - pkg_provided = visibility = system = ((), ()) - virtuals = {} - - -def incremental_expansion(orig, iterable, msg_prefix=''): - for i in iterable: - if i[0] == '-': - i = i[1:] - if not i: - raise ValueError("%sencountered an incomplete negation, '-'" - % msg_prefix) - orig.discard(i) - else: - orig.add(i) - - -class OnDiskProfile(object): - - pkgcore_config_type = ConfigHint({'basepath':'str', 'profile':'str', - 'incrementals':'list'}, required=('basepath', 'profile'), - typename='profile') - - def __init__(self, basepath, profile, incrementals=const.incrementals, - load_profile_base=True): - self.basepath = basepath - self.profile = profile - self.node = ProfileNode(pjoin(basepath, profile)) - self.incrementals = incrementals - self.load_profile_base = load_profile_base - - @property - def arch(self): - return self.default_env.get("ARCH") - - @property - def deprecated(self): - return self.node.deprecated - - def _load_stack(self): - def f(node): - for x in node.parents: - for y in f(x): - yield y - yield node - - l = list(f(self.node)) - if self.load_profile_base: - l = [EmptyRootNode(self.basepath)] + l - return tuple(l) - - def _collapse_use_dict(self, attr): - - stack = [getattr(x, attr) for x in self.stack] - - global_on = set() - puse_on = {} - puse_off = {} - atrue = packages.AlwaysTrue - for mapping in stack: - # process globals (use.(mask|force) first) - val = mapping.get(atrue) - if val is not None: - # global wipes everything affecting the flag thus far. - global_on.difference_update(val[0]) - for u in val[0]: - puse_on.pop(u, None) - puse_off.pop(u, None) - - # this *is* right; if it's global, it stomps everything prior. - global_on.update(val[1]) - for u in val[1]: - puse_on.pop(u, None) - puse_off.pop(u, None) - - # process less specific... - for key, val in mapping.iteritems(): - if key == atrue: - continue - for u in val[0]: - # is it even on? if not, don't level it. - if u in global_on: - if u not in puse_off: - puse_off[u] = set([key]) - else: - puse_off[u].add(key) - else: - s = puse_on.get(u) - if s is not None: - # chuck the previous override. - s.discard(key) - - for u in val[1]: - # if it's on already, no need to set it. - if u not in global_on: - if u not in puse_on: - puse_on[u] = set([key]) - else: - puse_on[u].add(key) - else: - s = puse_off.get(u) - if s is not None: - s.discard(key) - - # now we recompose it into a global on, and a stream of global trins. - d = {} - if global_on: - d[atrue] = set(global_on) - # reverse the mapping. - for data in (puse_on.iteritems(), - (("-"+k, v) for k,v in puse_off.iteritems())): - for use, key_set in data: - for key in key_set: - s = d.get(key) - if s is None: - d[key] = set([use]) - else: - s.add(use) - for k, v in d.iteritems(): - d[k] = tuple(v) - return d - - def _collapse_generic(self, attr): - s = set() - for node in self.stack: - val = getattr(node, attr) - s.difference_update(val[0]) - s.update(val[1]) - return s - - def _collapse_env(self): - d = {} - inc = self.incrementals - for profile in self.stack: - for key, val in profile.default_env.iteritems(): - if key in inc: - val = val.split() - s = d.get(key) - if s is None: - s = d[key] = set() - incremental_expansion(s, val, - "expanding %s make.defaults: " % profile) - if not s: - del d[key] - else: - d[key] = val - return d - - @property - def use_expand(self): - if "USE_EXPAND" in self.incrementals: - return tuple(self.default_env["USE_EXPAND"]) - return tuple(self.default_env["USE_EXPAND"].split()) - - @property - def use_expand_hidden(self): - if "USE_EXPAND_HIDDEN" in self.incrementals: - return tuple(self.default_env["USE_EXPAND_HIDDEN"]) - return tuple(self.default_env["USE_EXPAND_HIDDEN"].split()) - - def _collapse_virtuals(self): - d = {} - for profile in self.stack: - d.update(profile.virtuals) - self.virtuals = d - self.make_virtuals_repo = partial(AliasedVirtuals, d) - - def _collapse_pkg_provided(self): - d = {} - for pkg in self._collapse_generic("pkg_provided"): - d.setdefault(pkg.category, {}).setdefault(pkg.package, - []).append(pkg.fullver) - return util.SimpleTree(d, pkg_klass=PkgProvided) - - def _collapse_masks(self): - return frozenset(chain(self._collapse_generic("masks"), - self._collapse_generic("visibility"))) - - def __getattr__(self, attr): - if attr == "stack": - self.stack = obj = self._load_stack() - elif attr in ('forced_use', 'masked_use', 'pkg_use'): - obj = self._collapse_use_dict(attr) - setattr(self, attr, obj) - elif attr == 'bashrc': - obj = self.bashrc = tuple(x.bashrc - for x in self.stack if x.bashrc is not None) - elif attr == 'system': - obj = self.system = self._collapse_generic(attr) - elif attr == 'masks': - obj = self.masks = self._collapse_masks() - elif attr == 'default_env': - obj = self.default_env = self._collapse_env() - elif attr == 'virtuals': - self._collapse_virtuals() - obj = self.virtuals - elif attr == 'make_virtuals_repo': - self._collapse_virtuals() - obj = self.make_virtuals_repo - elif attr == 'provides_repo': - obj = self.provides_repo = self._collapse_pkg_provided() - elif attr == 'path': - obj = self.node.path - else: - raise AttributeError(attr) - return obj - - -class PkgProvided(ebuild_src.base): - - package_is_real = False - __inst_caching__ = True - - keywords = InvertedContains(()) - - def __init__(self, *a, **kwds): - # 'None' repo. - ebuild_src.base.__init__(self, None, *a, **kwds) - object.__setattr__(self, "use", []) - object.__setattr__(self, "data", {}) - - -class ForgetfulDict(dict): - - def __setitem__(self, key, attr): - return - - def update(self, other): - return - - -class AliasedVirtuals(virtual.tree): - - """ - repository generated from a profiles default virtuals - """ - - def __init__(self, virtuals, repo, *overrides): - """ - @param virtuals: dict of virtual -> providers - @param repo: L{pkgcore.ebuild.repository.UnconfiguredTree} parent repo - @keyword overrides: mapping of virtual pkgname -> matches to override defaults - """ - virtual.tree.__init__(self, livefs=False) - self._original_virtuals = virtuals - self._overrides = tuple(overrides) - if not overrides: - # no point in delaying. - self.packages._cache['virtuals'] = tuple(virtuals.iterkeys()) - self._virtuals = virtuals - self.aliased_repo = repo - self._versions_map = {} - - def _load_data(self): - self._virtuals = self._delay_apply_overrides(self._original_virtuals, - self._overrides) - self.packages._cache['virtual'] = tuple(self._virtuals.iterkeys()) - - @staticmethod - def _delay_apply_overrides(virtuals, overrides): - d = {} - for vtree in overrides: - for virt, provider in vtree.default_providers.iteritems(): - if virt in d: - d[virt] &= d[virt] & provider - else: - d[virt] = provider - - if not d: - return virtuals - for k, v in d.iteritems(): - if len(v) == 1: - d[k] = tuple(v)[0] - else: - d[k] = packages.OrRestriction(finalize=True, *v) - virtuals = virtuals.copy() - virtuals.update(d) - return virtuals - - def _get_versions(self, catpkg): - if catpkg[0] != "virtual": - raise KeyError("no %s package in this repository" % catpkg) - vers = set() - for pkg in self.aliased_repo.itermatch(self._virtuals[catpkg[1]]): - self._versions_map.setdefault(catpkg[1], {}).setdefault(pkg.fullver, []).append( - pkg.versioned_atom) - vers.add(pkg.fullver) - return tuple(vers) - - def _expand_vers(self, cp, ver): - return self._versions_map.get(cp[1], {}).get(ver, ()) - - def _fetch_metadata(self, pkg): - import pdb;pdb.set_trace() - data = self._virtuals[pkg.package] - if isinstance(data, atom.atom): - data = [data] - data = [atom.atom("=%s-%s" % (x.key, pkg.fullver)) for x in data] - if len(data) == 1: - return data[0] - return packages.OrRestriction(finalize=True, *data) diff --git a/pkgcore/ebuild/repo_objs.py b/pkgcore/ebuild/repo_objs.py deleted file mode 100644 index 00923b9..0000000 --- a/pkgcore/ebuild/repo_objs.py +++ /dev/null @@ -1,165 +0,0 @@ -# Copyright: 2006 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -""" -package class for buildable ebuilds -""" - -from snakeoil.currying import post_curry -from snakeoil.demandload import demandload -demandload(globals(), - 'snakeoil.xml:etree', - 'pkgcore.ebuild:digest', - 'snakeoil:mappings', - 'errno', -) - - -class Maintainer(object): - - """Data on a single maintainer. - - At least one of email and name is not C{None}. - - @type email: C{unicode} object or C{None} - @ivar email: email address. - @type name: C{unicode} object or C{None} - @ivar name: full name - @type description: C{unicode} object or C{None} - @ivar description: description of maintainership. - """ - - __slots__ = ('email', 'description', 'name') - - def __init__(self, email=None, name=None, description=None): - if email is None and name is None: - raise ValueError('need at least one of name and email') - self.email = email - self.name = name - self.description = description - - def __str__(self): - if self.name is not None: - if self.email is not None: - res = '%s <%s>' % (self.name, self.email) - else: - res = self.name - else: - res = self.email - if self.description is not None: - return '%s (%s)' % (res, self.description) - return res - - -class MetadataXml(object): - """metadata.xml parsed results - - attributes are set to -1 if unloaded, None if no entry, or the value - if loaded - """ - - __slots__ = ("__weakref__", "_maintainers", "_herds", "_longdescription", - "_source") - - def __init__(self, source): - self._source = source - - def _generic_attr(self, attr): - if self._source is not None: - self._parse_xml() - return getattr(self, attr) - - for attr in ("herds", "maintainers", "longdescription"): - locals()[attr] = property(post_curry(_generic_attr, "_"+attr)) - del attr - - def _parse_xml(self, source=None): - if source is None: - source = self._source.get_fileobj() - tree = etree.parse(source) - maintainers = [] - for x in tree.findall("maintainer"): - name = email = description = None - for e in x: - if e.tag == "name": - name = e.text - elif e.tag == "email": - email = e.text - elif e.tag == 'description': - description = e.text - maintainers.append(Maintainer( - name=name, email=email, description=description)) - - self._maintainers = tuple(maintainers) - self._herds = tuple(x.text for x in tree.findall("herd")) - - # Could be unicode! - longdesc = tree.findtext("longdescription") - if longdesc: - longdesc = ' '.join(longdesc.strip().split()) - self._longdescription = longdesc - self._source = None - - -class LocalMetadataXml(MetadataXml): - - __slots__ = () - - def _parse_xml(self): - try: - MetadataXml._parse_xml(self, open(self._source, "rb", 32768)) - except IOError, oe: - if oe.errno != errno.ENOENT: - raise - self._maintainers = () - self._herds = () - self._longdescription = None - self._source = None - - - -class Manifest(object): - - def __init__(self, source, enforce_gpg=False): - self._source = (source, not enforce_gpg) - - def _pull_manifest(self): - if self._source is None: - return - source, gpg = self._source - data = digest.parse_manifest(source, ignore_gpg=gpg, - kls_override=mappings.ImmutableDict) - self._dist, self._aux, self._ebuild, self._misc = data[0] - self._version = data[1] - self._source = None - - @property - def version(self): - self._pull_manifest() - return self._version - - @property - def required_files(self): - self._pull_manifest() - return mappings.StackedDict(self._ebuild, self._misc) - - @property - def aux_files(self): - self._pull_manifest() - return self._aux - - @property - def distfiles(self): - self._pull_manifest() - if self.version != 2: - raise TypeError("only manifest2 instances carry digest data") - return self._dist - - -class SharedPkgData(object): - - __slots__ = ("__weakref__", "metadata_xml", "manifest") - - def __init__(self, metadata_xml, manifest): - self.metadata_xml = metadata_xml - self.manifest = manifest diff --git a/pkgcore/ebuild/repository.py b/pkgcore/ebuild/repository.py deleted file mode 100644 index 6e39d97..0000000 --- a/pkgcore/ebuild/repository.py +++ /dev/null @@ -1,316 +0,0 @@ -# Copyright: 2005-2006 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -""" -ebuild repository, specific to gentoo ebuild trees (whether cvs or rsync) -""" - -import os, stat -from itertools import imap, ifilterfalse - -from pkgcore.repository import prototype, errors, configured, syncable -from pkgcore.ebuild import eclass_cache as eclass_cache_module -from pkgcore.config import ConfigHint -from pkgcore.plugin import get_plugin - -from snakeoil.fileutils import read_dict, iter_read_bash -from snakeoil import currying -from snakeoil.osutils import (listdir_files, readfile, listdir_dirs, pjoin, - readlines) -from snakeoil.containers import InvertedContains -from snakeoil.obj import make_kls -from snakeoil.weakrefs import WeakValCache - -from snakeoil.demandload import demandload -demandload(globals(), - 'pkgcore.ebuild.ebd:buildable', - 'pkgcore.interfaces.data_source:local_source', - 'pkgcore.ebuild:digest', - 'pkgcore.ebuild:repo_objs', - 'pkgcore.ebuild:atom', - 'random:shuffle', - 'errno', -) - - -metadata_offset = "profiles" - -class UnconfiguredTree(syncable.tree_mixin, prototype.tree): - - """ - raw implementation supporting standard ebuild tree. - - return packages don't have USE configuration bound to them. - """ - - false_packages = frozenset(["CVS", ".svn"]) - false_categories = frozenset([ - "eclass", "profiles", "packages", "distfiles", "metadata", - "licenses", "scripts", "CVS", "local"]) - configured = False - configurables = ("domain", "settings") - configure = None - format_magic = "ebuild_src" - enable_gpg = False - - pkgcore_config_type = ConfigHint( - {'location': 'str', 'cache': 'refs:cache', - 'eclass_cache': 'ref:eclass_cache', - 'default_mirrors': 'list', 'sync': 'lazy_ref:syncer', - 'override_repo_id':'str'}, - typename='repo') - - def __init__(self, location, cache=(), eclass_cache=None, - default_mirrors=None, sync=None, override_repo_id=None): - - """ - @param location: on disk location of the tree - @param cache: sequence of L{pkgcore.cache.template.database} instances - to use for storing metadata - @param eclass_cache: If not None, L{pkgcore.ebuild.eclass_cache} - instance representing the eclasses available, - if None, generates the eclass_cache itself - @param default_mirrors: Either None, or sequence of mirrors to try - fetching from first, then falling back to other uri - """ - - prototype.tree.__init__(self) - syncable.tree_mixin.__init__(self, sync) - self._repo_id = override_repo_id - self.base = self.location = location - try: - if not stat.S_ISDIR(os.stat(self.base).st_mode): - raise errors.InitializationError( - "base not a dir: %s" % self.base) - - except OSError: - raise errors.InitializationError( - "lstat failed on base %s" % self.base) - if eclass_cache is None: - self.eclass_cache = eclass_cache_module.cache( - pjoin(self.base, "eclass"), self.base) - else: - self.eclass_cache = eclass_cache - - fp = pjoin(self.base, metadata_offset, "thirdpartymirrors") - mirrors = {} - if os.path.exists(fp): - f = open(fp, "r") - try: - for k, v in read_dict(f, splitter=None, - source_isiter=True).iteritems(): - v = v.split() - shuffle(v) - mirrors[k] = v - finally: - f.close() - if isinstance(cache, (tuple, list)): - cache = tuple(cache) - else: - cache = (cache,) - - self.mirrors = mirrors - self.default_mirrors = default_mirrors - self.cache = cache - self.package_class = get_plugin("format." + self.format_magic)( - self, cache, self.eclass_cache, self.mirrors, self.default_mirrors) - self._shared_pkg_cache = WeakValCache() - - @property - def repo_id(self): - if self._repo_id is None: - # thank you spb for a stupid location, and stupid file name. - r = readfile(pjoin(self.location, "profiles", - "repo_name"), True) - if r is None: - self._repo_id = self.location - else: - self._repo_id = r.strip() - return self._repo_id - - def rebind(self, **kwds): - - """ - generate a new tree instance with the same location using new keywords. - - @param kwds: see __init__ for valid values - """ - - o = self.__class__(self.location, **kwds) - o.categories = self.categories - o.packages = self.packages - o.versions = self.versions - return o - - def _get_categories(self, *optional_category): - # why the auto return? current porttrees don't allow/support - # categories deeper then one dir. - if optional_category: - #raise KeyError - return () - - try: - # try reading $LOC/profiles/categories if it's available. - cats = readlines(pjoin(self.base, 'profiles', 'categories'), True, True, - True) - if cats is not None: - return tuple(imap(intern, cats)) - - return tuple(imap(intern, - ifilterfalse(self.false_categories.__contains__, - (x for x in listdir_dirs(self.base) if x[0:1] != ".") - ))) - except (OSError, IOError), e: - raise KeyError("failed fetching categories: %s" % str(e)) - - def _get_packages(self, category): - cpath = pjoin(self.base, category.lstrip(os.path.sep)) - try: - return tuple(ifilterfalse(self.false_packages.__contains__, - listdir_dirs(cpath))) - except (OSError, IOError), e: - raise KeyError("failed fetching packages for category %s: %s" % \ - (pjoin(self.base, category.lstrip(os.path.sep)), \ - str(e))) - - def _get_versions(self, catpkg): - cppath = pjoin(self.base, catpkg[0], catpkg[1]) - # 7 == len(".ebuild") - pkg = catpkg[-1] + "-" - lp = len(pkg) - try: - return tuple(x[lp:-7] for x in listdir_files(cppath) - if x[-7:] == '.ebuild' and x[:lp] == pkg) - except (OSError, IOError), e: - raise KeyError("failed fetching versions for package %s: %s" % \ - (pjoin(self.base, catpkg.lstrip(os.path.sep)), str(e))) - - def _get_ebuild_path(self, pkg): - return pjoin(self.base, pkg.category, pkg.package, \ - "%s-%s.ebuild" % (pkg.package, pkg.fullver)) - - def _get_ebuild_src(self, pkg): - return local_source(self._get_ebuild_path(pkg)) - - def _get_shared_pkg_data(self, category, package): - key = (category, package) - o = self._shared_pkg_cache.get(key) - if o is None: - mxml = self._get_metadata_xml(category, package) - manifest = self._get_manifest(category, package) - o = repo_objs.SharedPkgData(mxml, manifest) - self._shared_pkg_cache[key] = o - return o - - def _get_metadata_xml(self, category, package): - return repo_objs.LocalMetadataXml(pjoin(self.base, category, - package, "metadata.xml")) - - def _get_manifest(self, category, package): - return repo_objs.Manifest(pjoin(self.base, category, package, - "Manifest"), enforce_gpg=self.enable_gpg) - - def _get_digests(self, pkg, force_manifest1=False): - manifest = pkg._shared_pkg_data.manifest - if manifest.version == 2 and not force_manifest1: - return manifest.distfiles - return digest.parse_digest(pjoin( - os.path.dirname(self._get_ebuild_path(pkg)), "files", - "digest-%s-%s" % (pkg.package, pkg.fullver))) - - def __str__(self): - return "%s.%s: location %s" % ( - self.__class__.__module__, self.__class__.__name__, self.base) - - def __repr__(self): - return "<ebuild %s location=%r @%#8x>" % (self.__class__.__name__, - self.base, id(self)) - - def _visibility_limiters(self): - try: - return [atom.atom(x.strip()) - for x in iter_read_bash( - pjoin(self.base, "profiles", "package.mask"))] - except IOError, i: - if i.errno != errno.ENOENT: - raise - del i - return [] - - -class SlavedTree(UnconfiguredTree): - - """ - repository that pulls repo metadata from a parent repo; mirrors - being the main metadata pulled at this point - """ - - orig_hint = UnconfiguredTree.pkgcore_config_type - d = dict(orig_hint.types.iteritems()) - d["parent_repo"] = 'ref:repo' - pkgcore_config_type = orig_hint.clone(types=d, - required=list(orig_hint.required) + ["parent_repo"], - positional=list(orig_hint.positional) + ["parent_repo"]) - del d, orig_hint - - def __init__(self, parent_repo, *args, **kwds): - UnconfiguredTree.__init__(self, *args, **kwds) - for k, v in parent_repo.mirrors.iteritems(): - if k not in self.mirrors: - self.mirrors[k] = v - self.package_class = get_plugin("format." + self.format_magic)( - self, self.cache, self.eclass_cache, self.mirrors, - self.default_mirrors) - - -class ConfiguredTree(configured.tree): - - """ - wrapper around a L{UnconfiguredTree} binding build/configuration data (USE) - """ - - configurable = "use" - config_wrappables = dict( - (x, currying.alias_class_method("evaluate_depset")) - for x in ["depends", "rdepends", "post_rdepends", "fetchables", - "license", "src_uri", "license", "provides"]) - - def __init__(self, raw_repo, domain, domain_settings, fetcher=None): - """ - @param raw_repo: L{UnconfiguredTree} instance - @param domain_settings: environment settings to bind - @param fetcher: L{pkgcore.fetch.base.fetcher} instance to use - for getting access to fetchable files - """ - if "USE" not in domain_settings: - raise errors.InitializationError( - "%s requires the following settings: 'USE', not supplied" % ( - self.__class__,)) - - configured.tree.__init__(self, raw_repo, self.config_wrappables) - self._get_pkg_use = domain.get_package_use - self.domain_settings = domain_settings - if fetcher is None: - self.fetcher = self.domain_settings["fetcher"] - else: - self.fetcher = fetcher - self._delayed_iuse = currying.partial(make_kls(InvertedContains), - InvertedContains) - - def _get_delayed_immutable(self, pkg, immutable): - return InvertedContains(pkg.iuse.difference(immutable)) - - def _get_pkg_kwds(self, pkg): - immutable, enabled = self._get_pkg_use(pkg) - return { - "initial_settings": enabled, - "unchangable_settings": self._delayed_iuse( - self._get_delayed_immutable, pkg, immutable), - "build_callback":self.generate_buildop} - - def generate_buildop(self, pkg, **kwds): - return buildable(pkg, self.domain_settings, pkg.repo.eclass_cache, - self.fetcher, **kwds) - -UnconfiguredTree.configure = ConfiguredTree -tree = UnconfiguredTree diff --git a/pkgcore/ebuild/resolver.py b/pkgcore/ebuild/resolver.py deleted file mode 100644 index a6cd78d..0000000 --- a/pkgcore/ebuild/resolver.py +++ /dev/null @@ -1,136 +0,0 @@ -# Copyright: 2006 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -""" -resolver configuration to match portage behaviour (misbehaviour in a few spots) -""" - -__all__ = ["upgrade_resolver", "min_install_resolver"] - -from pkgcore.repository import virtual -from pkgcore.repository.misc import nodeps_repo -from pkgcore.resolver import plan - -from snakeoil.demandload import demandload -demandload(globals(), - 'pkgcore.restrictions:packages,values', - 'pkgcore.pkgsets.glsa:KeyedAndRestriction', -) - -def upgrade_resolver(vdb, dbs, verify_vdb=True, nodeps=False, - force_replacement=False, - resolver_cls=plan.merge_plan, **kwds): - - """ - generate and configure a resolver for upgrading all processed nodes. - - @param vdb: list of L{pkgcore.repository.prototype.tree} instances - that represents the livefs - @param dbs: list of L{pkgcore.repository.prototype.tree} instances - representing sources of pkgs - @param verify_vdb: should we stop resolving once we hit the vdb, - or do full resolution? - @param force_vdb_virtuals: old style portage virtuals (non metapkgs) - cannot be technically sorted since their versions are from multiple - packages bleeding through- results make no sense essentially. - You want this option enabled if you're dealing in old style virtuals. - @return: L{pkgcore.resolver.plan.merge_plan} instance - """ - - f = plan.merge_plan.prefer_highest_version_strategy - # hack. - vdb = list(vdb.trees) - if not isinstance(dbs, (list, tuple)): - dbs = [dbs] - if nodeps: - vdb = map(nodeps_repo, vdb) - dbs = map(nodeps_repo, dbs) - elif not verify_vdb: - vdb = map(nodeps_repo, vdb) - - if force_replacement: - resolver_cls = generate_replace_resolver_kls(resolver_cls) - return resolver_cls(dbs + vdb, plan.pkg_sort_highest, f, **kwds) - - -def min_install_resolver(vdb, dbs, verify_vdb=True, force_vdb_virtuals=True, - force_replacement=False, resolver_cls=plan.merge_plan, - nodeps=False, **kwds): - """ - Resolver that tries to minimize the number of changes while installing. - - generate and configure a resolver that is focused on just - installing requests- installs highest version it can build a - solution for, but tries to avoid building anything not needed - - @param vdb: list of L{pkgcore.repository.prototype.tree} instances - that represents the livefs - @param dbs: list of L{pkgcore.repository.prototype.tree} instances - representing sources of pkgs - @param verify_vdb: should we stop resolving once we hit the vdb, - or do full resolution? - @param force_vdb_virtuals: old style portage virtuals (non metapkgs) - cannot be technically sorted since their versions are from multiple - packages bleeding through- results make no sense essentially. - You want this option enabled if you're dealing in old style virtuals. - @return: L{pkgcore.resolver.plan.merge_plan} instance - """ - - # nothing fancy required for force_vdb_virtuals, we just silently ignore it. - vdb = list(vdb.trees) - if not isinstance(dbs, (list, tuple)): - dbs = [dbs] - if nodeps: - vdb = map(nodeps_repo, vdb) - dbs = map(nodeps_repo, dbs) - elif not verify_vdb: - vdb = map(nodeps_repo, vdb) - - if force_replacement: - resolver_cls = generate_replace_resolver_kls(resolver_cls) - return resolver_cls(vdb + dbs, plan.pkg_sort_highest, - plan.merge_plan.prefer_reuse_strategy, **kwds) - -_vdb_restrict = packages.OrRestriction( - packages.PackageRestriction("repo.livefs", values.EqualityMatch(False)), - packages.AndRestriction( - packages.PackageRestriction( - "category", values.StrExactMatch("virtual")), - packages.PackageRestriction( - "package_is_real", values.EqualityMatch(False)) - ) - ) - -class empty_tree_merge_plan(plan.merge_plan): - - _vdb_restriction = _vdb_restrict - - def __init__(self, *args, **kwds): - """ - @param args: see L{pkgcore.resolver.plan.merge_plan.__init__} - for valid args - @param kwds: see L{pkgcore.resolver.plan.merge_plan.__init__} - for valid args - """ - plan.merge_plan.__init__(self, *args, **kwds) - # XXX *cough*, hack. - self._empty_dbs = self.dbs - - def add_atom(self, atom): - return plan.merge_plan.add_atom( - self, atom, dbs=self._empty_dbs) - - -def generate_replace_resolver_kls(resolver_kls): - - - class replace_resolver(resolver_kls): - overriding_resolver_kls = resolver_kls - _vdb_restriction = _vdb_restrict - - def add_atom(self, atom, **kwds): - return self.overriding_resolver_kls.add_atom( - self, KeyedAndRestriction( - self._vdb_restriction, atom, key=atom.key), **kwds) - - return replace_resolver diff --git a/pkgcore/ebuild/triggers.py b/pkgcore/ebuild/triggers.py deleted file mode 100644 index 0964f28..0000000 --- a/pkgcore/ebuild/triggers.py +++ /dev/null @@ -1,458 +0,0 @@ -# Copyright: 2006-2007 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -""" -gentoo/ebuild specific triggers -""" - -import os, errno -from pkgcore.merge import triggers, const, errors -from pkgcore.fs import livefs -from pkgcore.restrictions import values - -from snakeoil.osutils import normpath -from snakeoil.fileutils import read_bash_dict, AtomicWriteFile -from snakeoil.osutils import listdir_files -from snakeoil.lists import stable_unique, iflatten_instance -from snakeoil.osutils import join as pjoin - -from snakeoil.demandload import demandload -demandload(globals(), "fnmatch", 'pkgcore:os_data') - -colon_parsed = frozenset( - ["ADA_INCLUDE_PATH", "ADA_OBJECTS_PATH", "INFODIR", "INFOPATH", - "LDPATH", "MANPATH", "PATH", "PRELINK_PATH", "PRELINK_PATH_MASK", - "PYTHONPATH", "PKG_CONFIG_PATH", "ROOTPATH"]) - -incrementals = frozenset( - ['ADA_INCLUDE_PATH', 'ADA_OBJECTS_PATH', 'CLASSPATH', 'CONFIG_PROTECT', - 'CONFIG_PROTECT_MASK', 'INFODIR', 'INFOPATH', 'KDEDIRS', 'LDPATH', - 'MANPATH', 'PATH', 'PRELINK_PATH', 'PRELINK_PATH_MASK', 'PYTHONPATH', - 'ROOTPATH', 'PKG_CONFIG_PATH']) - -default_ldpath = ('/lib', '/lib64', '/lib32', - '/usr/lib', '/usr/lib64', '/usr/lib32') - -def collapse_envd(base): - collapsed_d = {} - try: - env_d_files = sorted(listdir_files(base)) - except OSError, oe: - if oe.errno != errno.ENOENT: - raise - else: - for x in env_d_files: - if x.endswith(".bak") or x.endswith("~") or x.startswith("._cfg") \ - or len(x) <= 2 or not x[0:2].isdigit(): - continue - d = read_bash_dict(pjoin(base, x)) - # inefficient, but works. - for k, v in d.iteritems(): - collapsed_d.setdefault(k, []).append(v) - del d - - loc_incrementals = set(incrementals) - loc_colon_parsed = set(colon_parsed) - - # split out env.d defined incrementals.. - # update incrementals *and* colon parsed for colon_seperated; - # incrementals on it's own is space seperated. - - for x in collapsed_d.pop("COLON_SEPARATED", []): - v = x.split() - if v: - loc_colon_parsed.update(v) - - loc_incrementals.update(loc_colon_parsed) - - # now space. - for x in collapsed_d.pop("SPACE_SEPARATED", []): - v = x.split() - if v: - loc_incrementals.update(v) - - # now reinterpret. - for k, v in collapsed_d.iteritems(): - if k not in loc_incrementals: - collapsed_d[k] = v[-1] - continue - if k in loc_colon_parsed: - collapsed_d[k] = filter(None, iflatten_instance( - x.split(':') for x in v)) - else: - collapsed_d[k] = filter(None, iflatten_instance( - x.split() for x in v)) - - return collapsed_d, loc_incrementals, loc_colon_parsed - - -def string_collapse_envd(envd_dict, incrementals, colon_incrementals): - """transform a passed in dict to strictly strings""" - for k, v in envd_dict.iteritems(): - if k not in incrementals: - continue - if k in colon_incrementals: - envd_dict[k] = ':'.join(v) - else: - envd_dict[k] = ' '.join(v) - - -def update_ldso(ld_search_path, offset='/'): - # we do an atomic rename instead of open and write quick - # enough (avoid the race iow) - fp = pjoin(offset, 'etc', 'ld.so.conf') - new_f = AtomicWriteFile(fp, uid=os_data.root_uid, gid=os_data.root_uid, perms=0644) - new_f.write("# automatically generated, edit env.d files instead\n") - new_f.writelines(x.strip()+"\n" for x in ld_search_path) - new_f.close() - - -class env_update(triggers.base): - - required_csets = () - _hooks = ('post_unmerge', 'post_merge') - _priority = 5 - - def trigger(self, engine): - offset = engine.offset - d, inc, colon = collapse_envd(pjoin(offset, "etc/env.d")) - - l = d.pop("LDPATH", None) - if l is not None: - update_ldso(l, engine.offset) - - string_collapse_envd(d, inc, colon) - - new_f = AtomicWriteFile(pjoin(offset, "etc", "profile.env"), uid=os_data.root_uid, gid=os_data.root_gid, perms=0644) - new_f.write("# autogenerated. update env.d instead\n") - new_f.writelines('export %s="%s"\n' % (k, d[k]) for k in sorted(d)) - new_f.close() - new_f = AtomicWriteFile(pjoin(offset, "etc", "profile.csh"), uid=os_data.root_uid, gid=os_data.root_gid, perms=0644) - new_f.write("# autogenerated, update env.d instead\n") - new_f.writelines('setenv %s="%s"\n' % (k, d[k]) for k in sorted(d)) - new_f.close() - - -def simple_chksum_compare(x, y): - found = False - for k, v in x.chksums.iteritems(): - if k == "size": - continue - o = y.chksums.get(k) - if o is not None: - if o != v: - return False - found = True - if "size" in x.chksums and "size" in y.chksums: - return x.chksums["size"] == y.chksums["size"] - return found - - -def gen_config_protect_filter(offset, extra_protects=(), extra_disables=()): - collapsed_d, inc, colon = collapse_envd(pjoin(offset, "etc/env.d")) - collapsed_d.setdefault("CONFIG_PROTECT", []).extend(extra_protects) - collapsed_d.setdefault("CONFIG_PROTECT_MASK", []).extend(extra_disables) - - r = [values.StrGlobMatch(normpath(x).rstrip("/") + "/") - for x in set(stable_unique(collapsed_d["CONFIG_PROTECT"] + ["/etc"]))] - if len(r) > 1: - r = values.OrRestriction(*r) - else: - r = r[0] - neg = stable_unique(collapsed_d["CONFIG_PROTECT_MASK"]) - if neg: - if len(neg) == 1: - r2 = values.StrGlobMatch(normpath(neg[0]).rstrip("/") + "/", - negate=True) - else: - r2 = values.OrRestriction( - negate=True, - *[values.StrGlobMatch(normpath(x).rstrip("/") + "/") - for x in set(neg)]) - r = values.AndRestriction(r, r2) - return r - - -class ConfigProtectInstall(triggers.base): - - required_csets = ('install_existing', 'install') - _hooks = ('pre_merge',) - _priority = 90 - - def __init__(self, extra_protects=(), extra_disables=()): - triggers.base.__init__(self) - self.renames = {} - self.extra_protects = extra_protects - self.extra_disables = extra_disables - - def register(self, engine): - triggers.base.register(self, engine) - t2 = ConfigProtectInstall_restore(self.renames) - t2.register(engine) - - def trigger(self, engine, existing_cset, install_cset): - # hackish, but it works. - protected_filter = gen_config_protect_filter(engine.offset, - self.extra_protects, self.extra_disables).match - protected = {} - - for x in existing_cset.iterfiles(): - if x.location.endswith("/.keep"): - continue - elif protected_filter(x.location): - replacement = install_cset[x] - if not simple_chksum_compare(replacement, x): - protected.setdefault( - pjoin(engine.offset, - os.path.dirname(x.location).lstrip(os.path.sep)), - []).append((os.path.basename(replacement.location), - replacement)) - - for dir_loc, entries in protected.iteritems(): - updates = dict((x[0], []) for x in entries) - try: - existing = sorted(x for x in os.listdir(dir_loc) - if x.startswith("._cfg")) - except OSError, oe: - if oe.errno != errno.ENOENT: - raise - # this shouldn't occur. - continue - - for x in existing: - try: - # ._cfg0000_filename - count = int(x[5:9]) - if x[9] != "_": - raise ValueError - fn = x[10:] - except (ValueError, IndexError): - continue - if fn in updates: - updates[fn].append((count, fn)) - - - # now we rename. - for fname, entry in entries: - # check for any updates with the same chksums. - count = 0 - for cfg_count, cfg_fname in updates[fname]: - if simple_chksum_compare(livefs.gen_obj( - pjoin(dir_loc, cfg_fname)), entry): - count = cfg_count - break - count = max(count, cfg_count + 1) - try: - install_cset.remove(entry) - except KeyError: - # this shouldn't occur... - continue - new_fn = pjoin(dir_loc, "._cfg%04i_%s" % (count, fname)) - new_entry = entry.change_attributes(location=new_fn) - install_cset.add(new_entry) - self.renames[new_entry] = entry - del updates - - -class ConfigProtectInstall_restore(triggers.base): - - required_csets = ('install',) - _hooks = ('post_merge',) - _priority = 10 - - def __init__(self, renames_dict): - triggers.base.__init__(self) - self.renames = renames_dict - - def trigger(self, engine, install_cset): - for new_entry, old_entry in self.renames.iteritems(): - try: - install_cset.remove(new_entry) - except KeyError: - continue - install_cset.add(old_entry) - self.renames.clear() - - -class ConfigProtectUninstall(triggers.base): - - required_csets = ('uninstall_existing', 'uninstall') - _hooks = ('pre_unmerge',) - - def trigger(self, engine, existing_cset, uninstall_cset): - protected_restrict = gen_config_protect_filter(engine.offset) - - remove = [] - for x in existing_cset.iterfiles(): - if x.location.endswith("/.keep"): - continue - if protected_restrict.match(x.location): - recorded_ent = uninstall_cset[x] - if not simple_chksum_compare(recorded_ent, x): - # chksum differs. file stays. - remove.append(recorded_ent) - - for x in remove: - del uninstall_cset[x] - - -class preinst_contents_reset(triggers.base): - - required_csets = ('install',) - _hooks = ('pre_merge',) - _priority = 1 - - def __init__(self, format_op): - triggers.base.__init__(self) - self.format_op = format_op - - def trigger(self, engine, cset): - # wipe, and get data again. - cset.clear() - cs = engine.new._parent.scan_contents(self.format_op.env["D"]) - if engine.offset != '/': - cs = cs.insert_offset(engine.offset) - cset.update(cs) - - -class collision_protect(triggers.base): - - required_csets = { - const.INSTALL_MODE:('install', 'install_existing'), - const.REPLACE_MODE:('install', 'install_existing', 'old_cset') - } - - _hooks = ('sanity_check',) - _engine_types = triggers.INSTALLING_MODES - - def __init__(self, extra_protects=(), extra_disables=()): - triggers.base.__init__(self) - self.extra_protects = extra_protects - self.extra_disables = extra_disables - - def trigger(self, engine, install, existing, old_cset=()): - if not existing: - return - - # for the moment, we just care about files - colliding = existing.difference(install.iterdirs()) - - # filter out daft .keep files. - - # hackish, but it works. - protected_filter = gen_config_protect_filter(engine.offset, - self.extra_protects, self.extra_disables).match - - l = [] - for x in colliding: - if x.location.endswith(".keep"): - l.append(x) - elif protected_filter(x.location): - l.append(x) - - colliding.difference_update(l) - del l, protected_filter - if not colliding: - return - - colliding.difference_update(old_cset) - if colliding: - raise errors.BlockModification(self, - "collision-protect: file(s) already exist: ( %s )" % - ', '.join(repr(x) for x in sorted(colliding))) - - -class InfoRegen(triggers.InfoRegen): - - _label = "ebuild info regen" - - def register(self, engine): - # wipe pre-existing info triggers. - for x in self._hooks: - if x not in engine.hooks: - continue - # yucky, but works. - wipes = [y for y in engine.hooks[x] - if y.label == triggers.InfoRegen._label] - for y in wipes: - engine.hooks[x].remove(y) - triggers.InfoRegen.register(self, engine) - - def trigger(self, engine, *args): - self.engine = engine - self.path = pjoin(engine.offset, "etc/env.d") - triggers.InfoRegen.trigger(self, engine, *args) - - @property - def locations(self): - collapsed_d = collapse_envd(self.path)[0] - l = collapsed_d.get("INFOPATH", ()) - if not l: - return triggers.InfoRegen.locations - elif isinstance(l, basestring): - l = l.split() - return l - - -class FixImageSymlinks(triggers.base): - required_csets = ('new_cset',) - _hooks = ('pre_merge',) - - def __init__(self, format_op): - triggers.base.__init__(self) - self.format_op = format_op - - def trigger(self, engine, cset): - d = self.format_op.env["D"].rstrip("/") + "/" - l = [x for x in cset.iterlinks() if x.target.startswith(d)] - if engine.observer: - o = engine.observer - for x in l: - o.warn("correcting %s sym pointing into $D: %s" % - (x.location, x.target)) - d_len = len(d) - - # drop the leading ${D}, and force an abspath via '/' - cset.update(x.change_attributes(target=pjoin('/', x.target[d_len:])) - for x in l) - - -def customize_engine(domain_settings, engine): - env_update().register(engine) - - protect = domain_settings.get('CONFIG_PROTECT', ()) - if isinstance(protect, basestring): - protect = protect.split() - mask = domain_settings.get('CONFIG_PROTECT_MASK', ()) - if isinstance(protect, basestring): - protect = protect.split() - - ConfigProtectInstall(protect, mask).register(engine) - ConfigProtectUninstall().register(engine) - - features = domain_settings.get("FEATURES", ()) - if "collision-protect" in features: - collision_protect(protect, mask).register(engine) - - install_mask = domain_settings.get("INSTALL_MASK", '').split() - - for x in ("man", "info", "doc"): - if "no%s" % x in features: - install_mask.append("/usr/share/%s" % x) - l = [] - for x in install_mask: - x = x.rstrip("/") - l.append(values.StrRegex(fnmatch.translate(x))) - l.append(values.StrRegex(fnmatch.translate("%s/*" % x))) - install_mask = l - - if install_mask: - if len(install_mask) == 1: - install_mask = install_mask[0] - else: - install_mask = values.OrRestriction(*install_mask) - triggers.PruneFiles(install_mask.match).register(engine) - # note that if this wipes all /usr/share/ entries, should - # wipe the empty dir. - - InfoRegen().register(engine) diff --git a/pkgcore/fetch/__init__.py b/pkgcore/fetch/__init__.py deleted file mode 100644 index c9065b7..0000000 --- a/pkgcore/fetch/__init__.py +++ /dev/null @@ -1,126 +0,0 @@ -# Copyright: 2005-2007 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -""" -functionality related to downloading files -""" - -from snakeoil.klass import generic_equality - -class fetchable(object): - - """class representing uri sources for a file and chksum information.""" - - __slots__ = ("filename", "uri", "chksums") - __attr_comparison__ = __slots__ - __metaclass__ = generic_equality - - def __init__(self, filename, uri=(), chksums=None): - """ - @param filename: filename... - @param uri: either None (no uri), - or a sequence of uri where the file is available - @param chksums: either None (no chksum data), - or a dict of chksum_type -> value for this file - """ - self.uri = uri - if chksums is None: - self.chksums = {} - else: - self.chksums = chksums - self.filename = filename - - def __str__(self): - return "('%s', '%s', (%s))" % ( - self.filename, self.uri, ', '.join(self.chksums)) - - def __repr__(self): - return "<%s filename=%r uri=%r chksums=%r @%#8x>" % ( - self.__class__.__name__, self.filename, self.uri, self.chksums, - id(self)) - - -class mirror(object): - """ - uri source representing a mirror tier - """ - __metaclass__ = generic_equality - __attr_comparison__ = ('mirror_name', 'mirrors') - - __slots__ = ("mirrors", "mirror_name") - - def __init__(self, mirrors, mirror_name): - """ - @param mirrors: list of hosts that comprise this mirror tier - @param mirror_name: name of the mirror tier - """ - - if not isinstance(mirrors, tuple): - mirrors = tuple(mirrors) - self.mirrors = mirrors - self.mirror_name = mirror_name - - def __iter__(self): - return iter(self.mirrors) - - def __str__(self): - return "mirror://%s" % self.mirror_name - - def __len__(self): - return len(self.mirrors) - - def __nonzero__(self): - return bool(self.mirrors) - - def __getitem__(self, idx): - return self.mirrors[idx] - - def __repr__(self): - return "<%s mirror tier=%r>" % (self.__class__, self.mirror_name) - - -class default_mirror(mirror): - pass - - -class uri_list(object): - - __slots__ = ("_uri_source", "filename", "__weakref__") - - def __init__(self, filename): - self._uri_source = [] - self.filename = filename - - def add_mirror(self, mirror_inst, suburi=None): - if not isinstance(mirror_inst, mirror): - raise TypeError("mirror must be a pkgcore.fetch.mirror instance") - if suburi is not None and '/' in suburi: - self._uri_source.append((mirror_inst, suburi.lstrip('/'))) - else: - self._uri_source.append(mirror_inst) - - def add_uri(self, uri): - self._uri_source.append(uri) - - def finalize(self): - self._uri_source = tuple(self._uri_source) - - def __iter__(self): - fname = self.filename - for entry in self._uri_source: - if isinstance(entry, basestring): - yield entry - elif isinstance(entry, tuple): - # mirror with suburi - for base_uri in entry[0]: - yield '%s/%s' % (base_uri.rstrip('/'), entry[1]) - else: - for base_uri in entry: - yield "%s/%s" % (base_uri.rstrip('/'), fname) - - def __str__(self): - return "file: %s, uri: %s" % (self.filename, - ', '.join(str(x) for x in self._uri_source)) - - def __nonzero__(self): - return bool(self._uri_source) diff --git a/pkgcore/fetch/base.py b/pkgcore/fetch/base.py deleted file mode 100644 index 8b86c9f..0000000 --- a/pkgcore/fetch/base.py +++ /dev/null @@ -1,80 +0,0 @@ -# Copyright: 2005-2007 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -""" -prototype fetcher class, all fetchers should derive from this -""" - -import os -from pkgcore.chksum import get_handlers, get_chksums -from pkgcore.fetch import errors - - -class fetcher(object): - - def _verify(self, file_location, target, all_chksums=True, handlers=None): - """ - internal function for derivatives. - - digs through chksums, and returns: - - -2: file doesn't exist. - - -1: if (size chksum is available, and - file is smaller than stated chksum) - - 0: if all chksums match - - 1: if file is too large (if size chksums are available) - or else size is right but a chksum didn't match. - - if all_chksums is True, all chksums must be verified; if false, all - a handler can be found for are used. - """ - - nondefault_handlers = handlers - if handlers is None: - handlers = get_handlers(target.chksums) - if all_chksums: - for x in target.chksums: - if x not in handlers: - raise errors.RequiredChksumDataMissing(target, x) - - if "size" in handlers: - val = handlers["size"](file_location) - if val is None: - return -2 - c = cmp(val, target.chksums["size"]) - if c: - if c < 0: - return -1 - return 1 - elif not os.path.exists(file_location): - return -2 - - chfs = set(target.chksums).intersection(handlers) - chfs.discard("size") - chfs = list(chfs) - if nondefault_handlers: - for x in chfs: - if not handlers[x](file_location) == target.chksums[x]: - return 1 - elif [target.chksums[x] for x in chfs] != \ - get_chksums(file_location, *chfs): - return 1 - - return 0 - - def __call__(self, *a, **kw): - return self.fetch(*a, **kw) - - def get_path(self, fetchable): - """ - return the on disk path to a fetchable if it's available, and fully - fetched. - - If it isn't, return None - """ - raise NotImplementedError(self.get_path) - - def get_storage_path(self): - """return the directory files are stored in - returns None if not applicable - """ - return None diff --git a/pkgcore/fetch/custom.py b/pkgcore/fetch/custom.py deleted file mode 100644 index 478a0dd..0000000 --- a/pkgcore/fetch/custom.py +++ /dev/null @@ -1,158 +0,0 @@ -# Copyright: 2005-2007 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -""" -fetcher class that pulls files via executing another program to do the fetching -""" - -import os -from pkgcore.spawn import spawn, is_userpriv_capable -from pkgcore.os_data import portage_uid, portage_gid -from pkgcore.fetch import errors, base, fetchable -from pkgcore.config import ConfigHint -from snakeoil.osutils import ensure_dirs, join as pjoin - -class MalformedCommand(errors.base): - - def __init__(self, command): - errors.base.__init__(self, - "fetchcommand is malformed: %s" % (command,)) - self.command = command - - -class fetcher(base.fetcher): - - pkgcore_config_type = ConfigHint( - {'userpriv': 'bool', 'required_chksums': 'list', - 'distdir': 'str', 'command': 'str', 'resume_command': 'str'}, - allow_unknowns=True) - - def __init__(self, distdir, command, resume_command=None, - required_chksums=None, userpriv=True, attempts=10, - readonly=False, **extra_env): - """ - @param distdir: directory to download files to - @type distdir: string - @param command: shell command to execute to fetch a file - @type command: string - @param resume_command: if not None, command to use for resuming- - if None, command is reused - @param required_chksums: if None, all chksums must be verified, - else only chksums listed - @type required_chksums: None or sequence - @param userpriv: depriv for fetching? - @param attempts: max number of attempts before failing the fetch - @param readonly: controls whether fetching is allowed - """ - base.fetcher.__init__(self) - self.distdir = distdir - if required_chksums is not None: - required_chksums = [x.lower() for x in required_chksums] - else: - required_chksums = [] - if len(required_chksums) == 1 and required_chksums[0] == "all": - self.required_chksums = None - else: - self.required_chksums = required_chksums - def rewrite_command(string): - new_command = string.replace("${DISTDIR}", self.distdir) - new_command = new_command.replace("$DISTDIR", self.distdir) - new_command = new_command.replace("${URI}", "%(URI)s") - new_command = new_command.replace("$URI", "%(URI)s") - new_command = new_command.replace("${FILE}", "%(FILE)s") - new_command = new_command.replace("$FILE", "%(FILE)s") - if new_command == string: - raise MalformedCommand(string) - try: - new_command % {"URI":"blah", "FILE":"blah"} - except KeyError, k: - raise Malformedcommand("%s: unexpected key %s" % (command, k.args[0])) - return new_command - - self.command = rewrite_command(command) - if resume_command is None: - self.resume_command = self.command - else: - self.resume_command = rewrite_command(resume_command) - - self.attempts = attempts - self.userpriv = userpriv - kw = {"mode":0775} - if readonly: - kw["mode"] = 0555 - if userpriv: - kw["gid"] = portage_gid - kw["minimal"] = True - if not ensure_dirs(self.distdir, **kw): - raise errors.distdirPerms( - self.distdir, "if userpriv, uid must be %i, gid must be %i. " - "if not readonly, directory must be 0775, else 0555" % ( - portage_uid, portage_gid)) - - self.extra_env = extra_env - - def fetch(self, target): - """ - fetch a file - - @type target: L{pkgcore.fetch.fetchable} instance - @return: None if fetching failed, - else on disk location of the copied file - """ - - - if not isinstance(target, fetchable): - raise TypeError( - "target must be fetchable instance/derivative: %s" % target) - - fp = pjoin(self.distdir, target.filename) - filename = os.path.basename(fp) - - uri = iter(target.uri) - if self.userpriv and is_userpriv_capable(): - extra = {"uid":portage_uid, "gid":portage_gid} - else: - extra = {} - extra["umask"] = 0002 - extra["env"] = self.extra_env - attempts = self.attempts - try: - while attempts >= 0: - c = self._verify(fp, target) - if c == 0: - return fp - elif c > 0: - try: - os.unlink(fp) - command = self.command - except OSError, oe: - raise errors.UnmodifiableFile(fp, oe) - elif c == -2: - command = self.command - else: - command = self.resume_command - - # yeah, it's funky, but it works. - if attempts > 0: - u = uri.next() - # note we're not even checking the results. the - # verify portion of the loop handles this. iow, - # don't trust their exit code. trust our chksums - # instead. - spawn(command % {"URI":u, "FILE":filename}, **extra) - attempts -= 1 - - except StopIteration: - # ran out of uris - return None - - return None - - def get_path(self, fetchable): - fp = pjoin(self.distdir, fetchable.filename) - if 0 == self._verify(fp, fetchable): - return fp - return None - - def get_storage_path(self): - return self.distdir diff --git a/pkgcore/fetch/errors.py b/pkgcore/fetch/errors.py deleted file mode 100644 index 1365089..0000000 --- a/pkgcore/fetch/errors.py +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright: 2005 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -""" -errors fetch subsystem may throw -""" - -class base(Exception): - pass - -class distdirPerms(base): - def __init__(self, distdir, required): - base.__init__( - self, "distdir '%s' required fs attributes weren't enforcable: %s" - % (distdir, required)) - self.distdir, self.required = distdir, required - -class UnmodifiableFile(base): - def __init__(self, filename, extra=''): - base.__init__(self, "Unable to update file %s, unmodifiable %s" - % (filename, extra)) - self.file = filename - -class UnknownMirror(base): - def __init__(self, host, uri): - base.__init__(self, "uri mirror://%s/%s is has no known mirror tier" - % (host, uri)) - self.host, self.uri = host, uri - -class RequiredChksumDataMissing(base): - def __init__(self, fetchable, chksum): - base.__init__(self, "chksum %s was configured as required, " - "but the data is missing from fetchable '%s'" - % (chksum, fetchable)) - self.fetchable, self.missing_chksum = fetchable, chksum diff --git a/pkgcore/fs/__init__.py b/pkgcore/fs/__init__.py deleted file mode 100644 index c2527ce..0000000 --- a/pkgcore/fs/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -# Copyright: 2006 Brian Harring <ferringb@gmail.com> -# License: GPL2 -# $Id:$ - -""" -filesystem abstractions, and select operations -""" diff --git a/pkgcore/fs/contents.py b/pkgcore/fs/contents.py deleted file mode 100644 index ec1281f..0000000 --- a/pkgcore/fs/contents.py +++ /dev/null @@ -1,279 +0,0 @@ -# Copyright: 2005-2007 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -""" -contents set- container of fs objects -""" - -from pkgcore.fs import fs -from snakeoil.compatibility import all -from snakeoil.klass import generic_equality -from snakeoil.demandload import demandload -from snakeoil.osutils import normpath -demandload(globals(), - 'pkgcore.fs.ops:offset_rewriter,change_offset_rewriter', -) -from itertools import ifilter -from operator import attrgetter - -def check_instance(obj): - if not isinstance(obj, fs.fsBase): - raise TypeError("'%s' is not a fs.fsBase deriviative" % obj) - return obj.location, obj - - -class contentsSet(object): - """set of L{fs<pkgcore.fs.fs>} objects""" - - __metaclass__ = generic_equality - __attr_comparison__ = ('_dict',) - - def __init__(self, initial=None, mutable=True): - - """ - @param initial: initial fs objs for this set - @type initial: sequence - @param mutable: controls if it modifiable after initialization - """ - self._dict = {} - if initial is not None: - self._dict.update(check_instance(x) for x in initial) - self.mutable = mutable - - def __str__(self): - return "%s([%s])" % (self.__class__.__name__, - ', '.join(str(x) for x in self)) - - def __repr__(self): - return "%s([%s])" % (self.__class__.__name__, - ', '.join(repr(x) for x in self)) - - def add(self, obj): - - """ - add a new fs obj to the set - - @param obj: must be a derivative of L{pkgcore.fs.fs.fsBase} - """ - - if not self.mutable: - # weird, but keeping with set. - raise AttributeError( - "%s is frozen; no add functionality" % self.__class__) - if not fs.isfs_obj(obj): - raise TypeError("'%s' is not a fs.fsBase class" % str(obj)) - self._dict[obj.location] = obj - - def __delitem__(self, obj): - - """ - remove a fs obj to the set - - @type obj: a derivative of L{pkgcore.fs.fs.fsBase} - or a string location of an obj in the set. - @raise KeyError: if the obj isn't found - """ - - if not self.mutable: - # weird, but keeping with set. - raise AttributeError( - "%s is frozen; no remove functionality" % self.__class__) - if fs.isfs_obj(obj): - del self._dict[obj.location] - else: - del self._dict[normpath(obj)] - - def remove(self, obj): - del self[obj] - - def discard(self, obj): - if fs.isfs_obj(obj): - self._dict.pop(obj.location, None) - else: - self._dict.pop(obj, None) - - def __getitem__(self, obj): - if fs.isfs_obj(obj): - return self._dict[obj.location] - return self._dict[normpath(obj)] - - def __contains__(self, key): - if fs.isfs_obj(key): - return key.location in self._dict - return normpath(key) in self._dict - - def clear(self): - """ - clear the set - @raise ttributeError: if the instance is frozen - """ - if not self.mutable: - # weird, but keeping with set. - raise AttributeError( - "%s is frozen; no clear functionality" % self.__class__) - self._dict.clear() - - @staticmethod - def _convert_loc(iterable): - f = fs.isfs_obj - for x in iterable: - if f(x): - yield x.location - else: - yield x - - @staticmethod - def _ensure_fsbase(iterable): - f = fs.isfs_obj - for x in iterable: - if not f(x): - raise ValueError("must be an fsBase derivative: got %r" % x) - yield x - - def difference(self, other): - if not hasattr(other, '__contains__'): - other = set(self._convert_loc(other)) - return contentsSet((x for x in self if x.location not in other), - mutable=self.mutable) - - def difference_update(self, other): - if not self.mutable: - raise TypeError("%r isn't mutable" % self) - - rem = self.remove - for x in other: - if x in self: - rem(x) - - def intersection(self, other): - return contentsSet((x for x in other if x in self), - mutable=self.mutable) - - def intersection_update(self, other): - if not self.mutable: - raise TypeError("%r isn't mutable" % self) - if not hasattr(other, '__contains__'): - other = set(self._convert_loc(other)) - - l = [x for x in self if x.location not in other] - for x in l: - self.remove(x) - - def issubset(self, other): - if not hasattr(other, '__contains__'): - other = set(self._convert_loc(other)) - return all(x.location in other for x in self._dict) - - def issuperset(self, other): - return all(x in self for x in other) - - def union(self, other): - c = contentsSet(other) - c.update(self) - return c - - def __iter__(self): - return self._dict.itervalues() - - def __len__(self): - return len(self._dict) - - def symmetric_difference(self, other): - c = contentsSet(mutable=True) - c.update(self) - c.symmetric_difference_update(other) - object.__setattr__(c, 'mutable', self.mutable) - return c - - def symmetric_difference_update(self, other): - if not self.mutable: - raise TypeError("%r isn't mutable" % self) - if not hasattr(other, '__contains__'): - other = contentsSet(self._ensure_fsbase(other)) - l = [] - for x in self: - if x in other: - l.append(x) - add = self.add - for x in other: - if x not in self: - add(x) - rem = self.remove - for x in l: - rem(x) - del l, rem - - def update(self, iterable): - self._dict.update((x.location, x) for x in iterable) - - def iterfiles(self, invert=False): - if invert: - return (x for x in self if not x.is_reg) - return ifilter(attrgetter('is_reg'), self) - - def files(self, invert=False): - return list(self.iterfiles(invert=invert)) - - def iterdirs(self, invert=False): - if invert: - return (x for x in self if not x.is_dir) - return ifilter(attrgetter('is_dir'), self) - - def dirs(self, invert=False): - return list(self.iterdirs(invert=invert)) - - def iterlinks(self, invert=False): - if invert: - return (x for x in self if not x.is_sym) - return ifilter(attrgetter('is_sym'), self) - - def links(self, invert=False): - return list(self.iterlinks(invert=invert)) - - def iterdevs(self, invert=False): - if invert: - return (x for x in self if not x.is_dev) - return ifilter(attrgetter('is_dev'), self) - - def devs(self, invert=False): - return list(self.iterdevs(invert=invert)) - - def iterfifos(self, invert=False): - if invert: - return (x for x in self if not x.is_fifo) - return ifilter(attrgetter('is_fifo'), self) - - def fifos(self, invert=False): - return list(self.iterfifos(invert=invert)) - - for k in ("files", "dirs", "links", "devs", "fifos"): - s = k.capitalize() - locals()[k].__doc__ = \ - """ - returns a list of just L{pkgcore.fs.fs.fs%s} instances - @param invert: if True, yield everything that isn't a - fs%s instance, else yields just fs%s - """ % (s.rstrip("s"), s, s) - locals()["iter"+k].__doc__ = \ - """ - a generator yielding just L{pkgcore.fs.fs.fs%s} instances - @param invert: if True, yield everything that isn't a - fs%s instance, else yields just fs%s - """ % (s.rstrip("s"), s, s) - del s - del k - - def clone(self, empty=False): - if empty: - return self.__class__([], mutable=True) - return self.__class__(self._dict.itervalues(), mutable=True) - - def insert_offset(self, offset): - cset = self.clone(empty=True) - cset.update(offset_rewriter(offset, self)) - return cset - - def change_offset(self, old_offset, new_offset): - cset = self.clone(empty=True) - cset.update(change_offset_rewriter(old_offset, new_offset, self)) - return cset diff --git a/pkgcore/fs/fs.py b/pkgcore/fs/fs.py deleted file mode 100644 index 811bad1..0000000 --- a/pkgcore/fs/fs.py +++ /dev/null @@ -1,285 +0,0 @@ -# Copyright 2004-2006 Brian Harring <ferringb@gmail.com> -# License: GPL - -""" -filesystem entry abstractions -""" - -import stat -from pkgcore.chksum import get_handlers, get_chksums -from os.path import sep as path_seperator, realpath, abspath -from pkgcore.interfaces.data_source import local_source -from snakeoil.mappings import LazyFullValLoadDict -from snakeoil.osutils import normpath - -# goofy set of classes representating the fs objects pkgcore knows of. - -__all__ = [ - "fsFile", "fsDir", "fsSymlink", "fsDev", "fsFifo"] -__all__.extend("is%s" % x for x in ("dir", "reg", "sym", "fifo", "dev", - "fs_obj")) - -# following are used to generate appropriate __init__, wiped from the -# namespace at the end of the module - -_fs_doc = { - "mode":"""@keyword mode: int, the mode of this entry. """ - """required if strict is set""", - "mtime":"""@keyword mtime: long, the mtime of this entry. """ - """required if strict is set""", - "uid":"""@keyword uid: int, the uid of this entry. """ - """required if strict is set""", - "gid":"""@keyword gid: int, the gid of this entry. """ - """required if strict is set""", -} - -def gen_doc_additions(init, slots): - if init.__doc__ is None: - d = raw_init_doc.split("\n") - else: - d = init.__doc__.split("\n") - init.__doc__ = "\n".join(k.lstrip() for k in d) + \ - "\n".join(_fs_doc[k] for k in _fs_doc if k in slots) - - -raw_init_doc = \ -""" -@param location: location (real or intended) for this entry -@param strict: is this fully representative of the entry, or only partially -@raise KeyError: if strict is enabled, and not all args are passed in -""" - -class fsBase(object): - - """base class, all extensions must derive from this class""" - __slots__ = ("location", "mtime", "mode", "uid", "gid") - __attrs__ = __slots__ - __default_attrs__ = {} - - locals().update((x.replace("is", "is_"), False) for x in - __all__ if x.startswith("is") and x.islower() and not - x.endswith("fs_obj")) - - def __init__(self, location, strict=True, **d): - - d["location"] = normpath(location) - - s = object.__setattr__ - if strict: - for k in self.__attrs__: - s(self, k, d[k]) - else: - for k, v in d.iteritems(): - s(self, k, v) - gen_doc_additions(__init__, __attrs__) - - def change_attributes(self, **kwds): - d = dict((x, getattr(self, x)) - for x in self.__attrs__ if hasattr(self, x)) - d.update(kwds) - # split location out - location = d.pop("location") - if not location.startswith(path_seperator): - location = abspath(location) - d["strict"] = False - return self.__class__(location, **d) - - def __setattr__(self, key, value): - raise AttributeError(key) - - def __getattr__(self, attr): - # we would only get called if it doesn't exist. - if attr in self.__attrs__: - return self.__default_attrs__.get(attr) - raise AttributeError(attr) - - def __hash__(self): - return hash(self.location) - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - return self.location == other.location - - def __ne__(self, other): - return not self == other - - def realpath(self, cache=None): - """calculate the abspath/canonicalized path for this entry, returning - a new instance if the path differs. - - @keyword cache: Either None (no cache), or a data object of path-> - resolved. Currently unused, but left in for forwards compatibility - """ - new_path = realpath(self.location) - if new_path == self.location: - return self - return self.change_attributes(location=new_path) - - -known_handlers = tuple(get_handlers()) - -class fsFile(fsBase): - - """file class""" - - __slots__ = ("chksums", "data_source") - __attrs__ = fsBase.__attrs__ + __slots__ - __default_attrs__ = {"mtime":0l} - - is_reg = True - - def __init__(self, location, chksums=None, data_source=None, **kwds): - """ - @param chksums: dict of checksums, key chksum_type: val hash val. - See L{pkgcore.chksum}. - """ - if "mtime" in kwds: - kwds["mtime"] = long(kwds["mtime"]) - if data_source is None: - data_source = local_source(location) - kwds["data_source"] = data_source - - if chksums is None: - # this can be problematic offhand if the file is modified - # but chksum not triggered - chksums = LazyFullValLoadDict(known_handlers, self._chksum_callback) - kwds["chksums"] = chksums - fsBase.__init__(self, location, **kwds) - gen_doc_additions(__init__, __slots__) - - def __repr__(self): - return "file:%s" % self.location - - def _chksum_callback(self, chfs): - return zip(chfs, get_chksums(self.data, *chfs)) - - @property - def data(self): - return self.data_source - - -class fsDir(fsBase): - - """dir class""" - - __slots__ = () - is_dir = True - - def __repr__(self): - return "dir:%s" % self.location - - def __cmp__(self, other): - return cmp( - self.location.split(path_seperator), - other.location.split(path_seperator)) - - -class fsLink(fsBase): - - """symlink class""" - - __slots__ = ("target",) - __attrs__ = fsBase.__attrs__ + __slots__ - is_sym = True - - def __init__(self, location, target, **kwargs): - """ - @param target: string, filepath of the symlinks target - """ - kwargs["target"] = target - fsBase.__init__(self, location, **kwargs) - gen_doc_additions(__init__, __slots__) - - def change_attributes(self, **kwds): - d = dict((x, getattr(self, x)) - for x in self.__attrs__ if hasattr(self, x)) - d.update(kwds) - # split location out - location = d.pop("location") - if not location.startswith(path_seperator): - location = abspath(location) - target = d.pop("target") - d["strict"] = False - return self.__class__(location, target, **d) - - def __repr__(self): - return "symlink:%s->%s" % (self.location, self.target) - - -fsSymlink = fsLink - - -class fsDev(fsBase): - - """dev class (char/block objects)""" - - __slots__ = ("major", "minor") - __attrs__ = fsBase.__attrs__ + __slots__ - __default_attrs__ = {"major":-1, "minor":-1} - is_dev = True - - def __init__(self, path, major=-1, minor=-1, **kwds): - if kwds.get("strict", True): - if major == -1 or minor == -1: - raise TypeError( - "major/minor must be specified and positive ints") - if not stat.S_IFMT(kwds["mode"]): - raise TypeError( - "mode %o: must specify the device type (got %o)" % ( - kwds["mode"], stat.S_IFMT(kwds["mode"]))) - kwds["major"] = major - kwds["minor"] = minor - else: - if major != -1: - major = int(major) - if major < 0: - raise TypeError( - "major/minor must be specified and positive ints") - kwds["major"] = major - - if minor != -1: - minor = int(minor) - if minor < 0: - raise TypeError( - "major/minor must be specified and positive ints") - kwds["minor"] = minor - - fsBase.__init__(self, path, **kwds) - - def __repr__(self): - return "device:%s" % self.location - - -def get_major_minor(stat_inst): - """get major/minor from a stat instance - @return: major,minor tuple of ints - """ - return ( stat_inst.st_rdev >> 8 ) & 0xff, stat_inst.st_rdev & 0xff - - -class fsFifo(fsBase): - - """fifo class (socket objects)""" - - __slots__ = () - is_fifo = True - - def __repr__(self): - return "fifo:%s" % self.location - -def mk_check(target, name): - def f(obj): - return isinstance(obj, target) - f.__name__ = name - f.__doc__ = "return True if obj is an instance of L{%s}, else False" % target.__name__ - return f - -isdir = mk_check(fsDir, 'isdir') -isreg = mk_check(fsFile, 'isreg') -issym = mk_check(fsSymlink, 'issym') -isfifo = mk_check(fsFifo, 'isfifo') -isdev = mk_check(fsDev, 'isdev') -isfs_obj = mk_check(fsBase, 'isfs_obj') - -del raw_init_doc, gen_doc_additions, _fs_doc, mk_check diff --git a/pkgcore/fs/livefs.py b/pkgcore/fs/livefs.py deleted file mode 100644 index 7bc445a..0000000 --- a/pkgcore/fs/livefs.py +++ /dev/null @@ -1,154 +0,0 @@ -# Copyright: 2005-2006 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -""" -interaction with the livefs: generating fs objects to represent the livefs. -""" - -import os, collections, errno -from stat import S_IMODE, S_ISDIR, S_ISREG, S_ISLNK, S_ISFIFO - -from pkgcore.fs.fs import ( - fsFile, fsDir, fsSymlink, fsDev, fsFifo, get_major_minor) -from pkgcore.fs.contents import contentsSet -from pkgcore.chksum import get_handlers -from pkgcore.interfaces.data_source import local_source - -from snakeoil.osutils import normpath, join as pjoin -from snakeoil.mappings import LazyValDict -from snakeoil.osutils import listdir - -__all__ = ["gen_obj", "scan", "iter_scan"] - - -def gen_chksums(handlers, location): - def f(key): - return handlers[key](location) - return LazyValDict(handlers, f) - - -def gen_obj(path, stat=None, chksum_handlers=None, real_location=None): - - """ - given a fs path, and an optional stat, create an appropriate fs obj. - - @param stat: stat object to reuse if available - @param real_location: real path to the object if path is the desired - location, rather then existant location. - @raise KeyError: if no obj type matches the stat checks - @return: L{pkgcore.fs.fs.fsBase} derivative - """ - - if real_location is None: - real_location = path - if stat is None: - stat = os.lstat(real_location) - if chksum_handlers is None: - chksum_handlers = get_handlers() - - mode = stat.st_mode - d = {"mtime":stat.st_mtime, "mode":S_IMODE(mode), - "uid":stat.st_uid, "gid":stat.st_gid} - if S_ISDIR(mode): - return fsDir(path, **d) - elif S_ISREG(mode): - d["size"] = stat.st_size - d["data_source"] = local_source(real_location) - return fsFile(path, **d) - elif S_ISLNK(mode): - d["target"] = os.readlink(real_location) - return fsSymlink(path, **d) - elif S_ISFIFO(mode): - return fsFifo(path, **d) - else: - major, minor = get_major_minor(stat) - d["minor"] = minor - d["major"] = major - d["mode"] = mode - return fsDev(path, **d) - - -# hmm. this code is roughly 25x slower then find. -# make it less slow somehow. the obj instantiation is a bit of a -# killer I'm afraid; without obj, looking at 2.3ms roughly best of 3 -# 100 iterations, obj instantiation, 58ms. -# also, os.path.join is rather slow. -# in this case, we know it's always pegging one more dir on, so it's -# fine doing it this way (specially since we're relying on -# os.path.sep, not '/' :P) - -def _internal_iter_scan(path, chksum_handlers): - dirs = collections.deque([normpath(path)]) - yield gen_obj(dirs[0], chksum_handlers=chksum_handlers) - while dirs: - base = dirs.popleft() - for x in listdir(base): - path = pjoin(base, x) - o = gen_obj(path, chksum_handlers=chksum_handlers, - real_location=path) - yield o - if isinstance(o, fsDir): - dirs.append(path) - - -def _internal_offset_iter_scan(path, chksum_handlers, offset): - offset = normpath(offset) - path = normpath(path) - dirs = collections.deque([path[len(offset):]]) - if dirs[0]: - yield gen_obj(dirs[0], chksum_handlers=chksum_handlers) - - sep = os.path.sep - while dirs: - base = dirs.popleft() - real_base = pjoin(offset, base.lstrip(sep)) - base = base.rstrip(sep) + sep - for x in listdir(real_base): - path = pjoin(base, x) - o = gen_obj(path, chksum_handlers=chksum_handlers, - real_location=pjoin(real_base, x)) - yield o - if isinstance(o, fsDir): - dirs.append(path) - - -def iter_scan(path, offset=None): - """ - Recursively scan a path. - - Does not follow symlinks pointing at dirs, just merely yields an - obj representing said symlink - - @return: an iterator of L{pkgcore.fs.fs.fsBase} objects. - - @param path: str path of what directory to scan in the livefs - @param offset: if not None, prefix to strip from each objects location. - if offset is /tmp, /tmp/blah becomes /blah - """ - chksum_handlers = get_handlers() - - if offset is None: - return _internal_iter_scan(path, chksum_handlers) - return _internal_offset_iter_scan(path, chksum_handlers, offset) - - -def scan(*a, **kw): - """ - calls list(iter_scan(*a, **kw)) - Look at iter_scan for valid args - """ - mutable = kw.pop("mutable", True) - return contentsSet(iter_scan(*a, **kw), mutable=mutable) - - -def intersect(cset): - """generate the intersect of a cset and the livefs""" - f = gen_obj - for x in cset: - try: - yield f(x.location) - except OSError, oe: - if oe.errno != errno.ENOENT: - raise - del oe - diff --git a/pkgcore/fs/ops.py b/pkgcore/fs/ops.py deleted file mode 100644 index d72f934..0000000 --- a/pkgcore/fs/ops.py +++ /dev/null @@ -1,323 +0,0 @@ -# Copyright: 2005-2007 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -""" -default fs ops. - -Shouldn't be accessed directly for the most part, use -L{pkgcore.plugins} to get at these ops. -""" - -import os, errno - -from pkgcore.fs import contents, fs -from pkgcore.fs.livefs import gen_obj -from pkgcore.spawn import spawn -from pkgcore.const import COPY_BINARY -from pkgcore.plugin import get_plugin - -from snakeoil.currying import partial -from snakeoil.osutils import ensure_dirs, pjoin, normpath - -__all__ = [ - "merge_contents", "unmerge_contents", "default_ensure_perms", - "default_copyfile", "default_mkdir"] - -def default_ensure_perms(d1, d2=None): - - """Enforce a fs objects attributes on the livefs. - - Attributes enforced are permissions, mtime, uid, gid. - - @param d2: if not None, an fs object for what's on the livefs now - @raise OSError: if fs object attributes can't be enforced - @return: True on success, else an exception is thrown - """ - - m, o, g, t = d1.mode, d1.uid, d1.gid, d1.mtime - if o is None: - o = -1 - if g is None: - g = -1 - if d2 is None: - do_mode, do_chown, do_mtime = True, True, True - else: - - do_mode = False - try: - if fs.isdir(d1) and fs.isdir(d2): - # if it's preexisting, keep it's perms. - do_mode = False - else: - do_mode = (m is not None and m != d2.mode) - except AttributeError: - # yes. this _is_ stupid. vdb's don't always store all attributes - do_mode = False - - do_chown = False - try: - do_chown = (o != d2.uid or g != d2.gid) - except AttributeError: - do_chown = True - - try: - do_mtime = (t != d2.mtime) - except AttributeError: - do_mtime = True - - if do_chown and (o != -1 or g != -1): - os.lchown(d1.location, o, g) - if do_mode and m is not None: - os.chmod(d1.location, m) - if do_mtime and t is not None: - os.utime(d1.location, (t, t)) - return True - - -def default_mkdir(d): - """ - mkdir for a fsDir object - - @param d: L{pkgcore.fs.fs.fsDir} instance - @raise OSError: if can't complete - @return: true if success, else an exception is thrown - """ - if not d.mode: - mode = 0777 - else: - mode = d.mode - os.mkdir(d.location, mode) - get_plugin("fs_ops.ensure_perms")(d) - return True - -# minor hack. - -class FailedCopy(TypeError): - - def __init__(self, obj, msg): - self.obj = obj - self.msg = msg - - def __str__(self): - return "failed copying %s:" % (self.obj, self.msg) - - -class CannotOverwrite(FailedCopy): - def __init__(self, obj, existing): - self.obj, self.existing = obj, existing - - def __str__(self): - return "cannot write %s due to %s existing" % ( - self.obj, self.existing) - - -def default_copyfile(obj, mkdirs=False): - """ - copy a L{fs obj<pkgcore.fs.fs.fsBase>} to it's stated location. - - @param obj: L{pkgcore.fs.fs.fsBase} instance, exempting fsDir - @raise OSError:, for non file objs, Exception (this needs to be fixed - @return: true if success, else an exception is thrown - """ - - existant = False - ensure_perms = get_plugin("fs_ops.ensure_perms") - if not fs.isfs_obj(obj): - raise TypeError("obj must be fsBase derivative: %r" % obj) - elif fs.isdir(obj): - raise TypeError("obj must not be a fsDir instance: %r" % obj) - - try: - existing = gen_obj(obj.location) - if fs.isdir(existing): - raise CannotOverwrite(obj, existing) - existant = True - except OSError, oe: - # verify the parent dir is there at least - basefp = os.path.dirname(obj.location) - if basefp.strip(os.path.sep) and not os.path.exists(basefp): - if mkdirs: - if not ensure_dirs(basefp, mode=0750, minimal=True): - raise FailedCopy(obj, str(oe)) - else: - raise - existant = False - - if not existant: - fp = obj.location - else: - fp = existant_fp = obj.location + "#new" - - if fs.isreg(obj): - src_f = obj.data.get_fileobj() - new_f = open(fp, "wb", 32768) - d = src_f.read(32768) - while d: - new_f.write(d) - d = src_f.read(32768) - new_f.close() - del src_f - elif fs.issym(obj): - os.symlink(obj.target, fp) - elif fs.isfifo(obj): - os.mkfifo(fp) - elif fs.isdev(obj): - dev = os.makedev(obj.major, obj.minor) - os.mknod(fp, obj.mode, dev) - else: - ret = spawn([COPY_BINARY, "-Rp", obj.location, fp]) - if ret != 0: - raise FailedCopy(obj, "got %i from %s -Rp" % ret) - if not fs.issym(obj): - ensure_perms(obj.change_attributes(location=fp)) - - if existant: - os.rename(existant_fp, obj.location) - return True - - -def offset_rewriter(offset, iterable): - sep = os.path.sep - for x in iterable: - yield x.change_attributes( - location=pjoin(offset, x.location.lstrip(sep))) - - -def change_offset_rewriter(orig_offset, new_offset, iterable): - offset_len = len(orig_offset.rstrip(os.path.sep)) - # localize it. - npf = normpath - for x in iterable: - # slip in the '/' default to force it to still generate a - # full path still - yield x.change_attributes( - location=npf(pjoin(new_offset, x.location[offset_len:]))) - - -def merge_contents(cset, offset=None, callback=lambda obj:None): - - """ - merge a L{pkgcore.fs.contents.contentsSet} instance to the livefs - - @param cset: L{pkgcore.fs.contents.contentsSet} instance - @param offset: if not None, offset to prefix all locations with. - Think of it as target dir. - @param callback: callable to report each entry being merged - @raise OSError: see L{default_copyfile} and L{default_mkdir} - @return: True, or an exception is thrown on failure - (OSError, although see default_copyfile for specifics). - """ - - ensure_perms = get_plugin("fs_ops.ensure_perms") - copyfile = get_plugin("fs_ops.copyfile") - mkdir = get_plugin("fs_ops.mkdir") - - if not isinstance(cset, contents.contentsSet): - raise TypeError("cset must be a contentsSet, got %r" % (cset,)) - - if offset is not None: - if os.path.exists(offset): - if not os.path.isdir(offset): - raise TypeError("offset must be a dir, or not exist: %s" % offset) - else: - mkdir(fs.fsDir(offset, strict=False)) - iterate = partial(offset_rewriter, offset.rstrip(os.path.sep)) - else: - iterate = iter - - d = list(iterate(cset.iterdirs())) - d.sort() - for x in d: - callback(x) - - try: - # we pass in the stat ourselves, using stat instead of - # lstat gen_obj uses internally; this is the equivalent of - # "deference that link" - obj = gen_obj(x.location, stat=os.stat(x.location)) - if not fs.isdir(obj): - raise Exception( - "%s exists and needs to be a dir, but is a %s" % - (x.location, obj)) - ensure_perms(x, obj) - except OSError, oe: - if oe.errno != errno.ENOENT: - raise - try: - # we do this form to catch dangling symlinks - mkdir(x) - except OSError, oe: - if oe.errno != errno.EEXIST: - raise - os.unlink(x.location) - mkdir(x) - ensure_perms(x) - del d - - # might look odd, but what this does is minimize the try/except cost - # to one time, assuming everything behaves, rather then per item. - i = iterate(cset.iterdirs(invert=True)) - while True: - try: - for x in i: - callback(x) - copyfile(x, mkdirs=True) - break - except CannotOverwrite, cf: - if not fs.issym(x): - raise - - # by this time, all directories should've been merged. - # thus we can check the target - try: - if not fs.isdir(gen_obj(pjoin(x.location, x.target))): - raise - except OSError: - raise cf - return True - - -def unmerge_contents(cset, offset=None, callback=lambda obj:None): - - """ - unmerge a L{pkgcore.fs.contents.contentsSet} instance to the livefs - - @param cset: L{pkgcore.fs.contents.contentsSet} instance - @param offset: if not None, offset to prefix all locations with. - Think of it as target dir. - @param callback: callable to report each entry being unmerged - @raise OSError: see L{default_copyfile} and L{default_mkdir} - @return: True, or an exception is thrown on failure - (OSError, although see default_copyfile for specifics). - """ - - iterate = iter - if offset is not None: - iterate = partial(offset_rewriter, offset.rstrip(os.path.sep)) - - for x in iterate(cset.iterdirs(invert=True)): - callback(x) - try: - os.unlink(x.location) - except OSError, e: - if e.errno != errno.ENOENT: - raise - # this is a fair sight faster then using sorted/reversed - l = list(iterate(cset.iterdirs())) - l.sort(reverse=True) - for x in l: - try: - os.rmdir(x.location) - except OSError, e: - if not e.errno in (errno.ENOTEMPTY, errno.ENOENT, errno.ENOTDIR, - errno.EBUSY): - raise - else: - callback(x) - return True - -# Plugin system priorities -for func in [default_copyfile, default_ensure_perms, default_mkdir, - merge_contents, unmerge_contents]: - func.priority = 1 -del func diff --git a/pkgcore/fs/tar.py b/pkgcore/fs/tar.py deleted file mode 100644 index 950c9bd..0000000 --- a/pkgcore/fs/tar.py +++ /dev/null @@ -1,127 +0,0 @@ -# Copyright: 2006 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -""" -binpkg tar utilities -""" -import os, stat -from pkgcore.fs.fs import fsFile, fsDir, fsSymlink, fsFifo, fsDev -from pkgcore.fs import contents -from pkgcore.interfaces.data_source import data_source - -from snakeoil.tar import tarfile -from snakeoil.mappings import OrderedDict, StackedDict -from snakeoil.currying import partial - -class tar_data_source(data_source): - def get_fileobj(self): - return self.data() - -class TarContentsSet(contents.contentsSet): - - def __init__(self, initial=None, mutable=False): - contents.contentsSet.__init__(self, mutable=True) - self._dict = OrderedDict() - if initial: - self.update(initial) - self.mutable = mutable - - -known_compressors = {"bz2": tarfile.TarFile.bz2open, - "gz": tarfile.TarFile.gzopen, - None: tarfile.TarFile.open} - -def write_set(contents_set, filepath, compressor='bz2'): - if compressor not in known_compressors: - raise ValueError("compression must be one of %r, got %r" % - (known_compressors.keys(), compressor)) - tar_fd = known_compressors[compressor](filepath, mode="w") - - # first add directories, then everything else - # this is just a pkgcore optimization, it prefers to see the dirs first. - dirs = contents_set.dirs() - dirs.sort() - for x in dirs: - tar_fd.addfile(fsobj_to_tarinfo(x)) - del dirs - for x in contents_set.iterdirs(invert=True): - t = fsobj_to_tarinfo(x) - if t.isreg(): - tar_fd.addfile(t, fileobj=x.data.get_fileobj()) - else: - tar_fd.addfile(t) - tar_fd.close() - -def tarinfo_to_fsobj(src_tar): - psep = os.path.sep - for member in src_tar: - d = { - "uid":member.uid, "gid":member.gid, - "mtime":member.mtime, "mode":member.mode} - location = psep + member.name.strip(psep) - if member.isdir(): - if member.name.strip(psep) == ".": - continue - yield fsDir(location, **d) - elif member.isreg(): - d["data_source"] = tar_data_source(partial( - src_tar.extractfile, member.name)) - # bit of an optimization; basically, we know size, so - # we stackdict it so that the original value is used, rather then - # triggering an full chksum run for size - f = fsFile(location, **d) - object.__setattr__(f, "chksums", StackedDict( - {"size":long(member.size)}, f.chksums)) - yield f - elif member.issym() or member.islnk(): - yield fsSymlink(location, member.linkname, **d) - elif member.isfifo(): - yield fsFifo(location, **d) - elif member.isdev(): - d["major"] = long(member.major) - d["minor"] = long(member.minor) - yield fsDev(location, **d) - else: - raise AssertionError( - "unknown type %r, %r was encounted walking tarmembers" % - (member, member.type)) - -def fsobj_to_tarinfo(fsobj): - t = tarfile.TarInfo() - if isinstance(fsobj, fsFile): - t.type = tarfile.REGTYPE - t.size = fsobj.chksums["size"] - elif isinstance(fsobj, fsDir): - t.type = tarfile.DIRTYPE - elif isinstance(fsobj, fsSymlink): - t.type = tarfile.SYMTYPE - t.linkname = fsobj.target - elif isinstance(fsobj, fsFifo): - t.type = tarfile.FIFOTYPE - elif isinstance(fsobj, fsDev): - if stat.S_ISCHR(fsobj.mode): - t.type = tarfile.CHRTYPE - else: - t.type = tarfile.BLKTYPE - t.devmajor = fsobj.major - t.devminor = fsobj.minor - t.name = fsobj.location - t.mode = fsobj.mode - t.uid = fsobj.uid - t.gid = fsobj.gid - t.mtime = fsobj.mtime - return t - -def generate_contents(path, compressor="bz2"): - """ - generate a contentset from a tarball - - @param path: string path to location on disk - @param compressor: defaults to bz2; decompressor to use, see - L{known_compressors} for list of valid compressors - """ - if compressor not in known_compressors: - raise ValueError("compressor needs to be one of %r, got %r" % - (known_compressors.keys(), compressor)) - t = known_compressors[compressor](path, mode="r") - return TarContentsSet(tarinfo_to_fsobj(t), mutable=False) diff --git a/pkgcore/interfaces/__init__.py b/pkgcore/interfaces/__init__.py deleted file mode 100644 index 57eb691..0000000 --- a/pkgcore/interfaces/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright: 2005 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -""" -interface templates for package/repository/data source objects -""" diff --git a/pkgcore/interfaces/data_source.py b/pkgcore/interfaces/data_source.py deleted file mode 100644 index abd4e4b..0000000 --- a/pkgcore/interfaces/data_source.py +++ /dev/null @@ -1,86 +0,0 @@ -# Copyright: 2005-2006 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -""" -data source. - -Think of it as a far more minimal form of file protocol -""" - -import StringIO -from snakeoil.currying import pre_curry - -def generic_immutable_method(attr, self, *a, **kwds): - raise AttributeError("%s doesn't have %s" % (self.__class__, attr)) - -class native_ro_StringIO(StringIO.StringIO): - locals().update([(k, pre_curry(generic_immutable_method, k)) for k in - ["write", "writelines", "truncate"]]) - -del generic_immutable_method - -class write_StringIO(StringIO.StringIO): - - def __init__(self, callback, *args, **kwds): - if not callable(callback): - raise TypeError("callback must be callable") - StringIO.StringIO.__init__(self, *args, **kwds) - self._callback = callback - - def close(self): - self.flush() - if self._callback is not None: - self.seek(0) - self._callback(self.read()) - self._callback = None - StringIO.StringIO.close(self) - -try: - import cStringIO - read_StringIO = cStringIO.StringIO -except ImportError: - read_StringIO = native_ro_StringIO - -class base(object): - """base class, all implementations should match this protocol""" - get_fileobj = get_path = None - - -class local_source(base): - - """locally accessible data source""" - - __slots__ = ("path", "mutable") - - def __init__(self, path, mutable=False): - """@param path: file path of the data source""" - base.__init__(self) - self.path = path - self.mutable = mutable - - def get_path(self): - return self.path - - def get_fileobj(self): - if self.mutable: - return open(self.path, "rb+", 32768) - return open(self.path, "rb", 32768) - - -class data_source(base): - - def __init__(self, data, mutable=False): - """@param data: data to wrap""" - base.__init__(self) - self.data = data - self.mutable = mutable - - get_path = None - - def get_fileobj(self): - if self.mutable: - return write_StringIO(self._reset_data, self.data) - return read_StringIO(self.data) - - def _reset_data(self, data): - self.data = data diff --git a/pkgcore/interfaces/format.py b/pkgcore/interfaces/format.py deleted file mode 100644 index f9456c8..0000000 --- a/pkgcore/interfaces/format.py +++ /dev/null @@ -1,195 +0,0 @@ -# Copyright: 2005-2007 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -""" -build operation -""" - -from snakeoil.dependant_methods import ForcedDepends - -__all__ = ('build_base', 'base', 'install', 'uninstall', 'replace', 'fetch', - 'empty_build_op', 'FailedDirectory', 'GenericBuildError', 'errors') - - -def _raw_fetch(self): - if not "files" in self.__dict__: - self.files = {} - - # this is being anal, but protect against pkgs that don't collapse - # common uri down to a single file. - gotten_fetchables = set(x.filename for x in self.files.values()) - for x in self.fetchables: - if x.filename in gotten_fetchables: - continue - fp = self.fetcher(x) - if fp is None: - return False - self.files[fp] = x - gotten_fetchables.add(x.filename) - return True - - -class maintenance(object): - stage_depends = {} - - __metaclass__ = ForcedDepends - - def __init__(self, pkg, observer=None): - self.obvserver = observer - self.pkg = pkg - - def config(self): - return True - - -class build_base(object): - stage_depends = {} - - __metaclass__ = ForcedDepends - - def __init__(self, observer=None): - self.observer = observer - - -class build(build_base): - stage_depends = { - "setup":"start", - "unpack":("fetch", "setup"), - "configure":"unpack", - "compile":"configure", - "test":"compile", - "install":"test", - "finalize":"install"} - - def setup(self): - return True - - fetch = _raw_fetch - - def unpack(self): - return True - - def configure(self): - return True - - def compile(self): - return True - - def test(self): - return True - - def install(self): - return True - - def finalize(self): - """finalize any build steps required""" - return True - - def cleanup(self): - """cleanup any working files/dirs created during building""" - return True - - for k in ( - "setup", "fetch", "unpack", "configure", "compile", "test", "install"): - locals()[k].__doc__ = ( - "execute any %s steps required; " - "implementations of this interface should overide this as needed" - % k) - for k in ( - "setup", "fetch", "unpack", "configure", "compile", "test", "install", - "finalize"): - o = locals()[k] - o.__doc__ = "\n".join(x.lstrip() for x in o.__doc__.split("\n") + [ - "@return: True on success, False on failure"]) - del o, k - - -class install(build_base): - stage_depends = {"preinst":"start", "postinst":"preinst", "finalize":"postinst"} - - def preinst(self): - """any pre merge steps needed""" - return True - - def postinst(self): - """any post merge steps needed""" - return True - - def finalize(self): - """finalize any merge steps required""" - return True - - -class uninstall(build_base): - stage_depends = {"prerm":"start", "postrm":"prerm", "finalize":"postrm"} - - def prerm(self): - """any pre unmerge steps needed""" - return True - - def postinst(self): - """any post unmerge steps needed""" - return True - - def finalize(self): - """finalize any unmerge steps required""" - return True - -class replace(install, uninstall): - - stage_depends = {"finalize":"postinst", "postinst":"postrm", - "postrm":"prerm", "prerm":"preinst", "preinst":"start"} - - -class fetch(object): - __metaclass__ = ForcedDepends - - stage_depends = {"finalize":"fetch"} - - fetch = _raw_fetch - - def __init__(self, pkg): - self.pkg = pkg - self.fetchables = pkg.fetchables - - def finalize(self): - """finalize any build steps required""" - return self.pkg - - def cleanup(self): - return True - - -class empty_build_op(build_base): - - stage_depends = {} - -# __metaclass__ = ForcedDepends - - def __init__(self, pkg, observer=None, clean=False): - build_base.__init__(self, observer) - self.pkg = pkg - - def cleanup(self): - return True - - def finalize(self): - return self.pkg - - -class BuildError(Exception): - pass - -class FailedDirectory(BuildError): - def __init__(self, path, text): - BuildError.__init__( - self, "failed creating/ensuring dir %s: %s" % (path, text)) - - -class GenericBuildError(BuildError): - def __init__(self, err): - BuildError.__init__(self, "Failed build operation: %s" % (err,)) - self.err = str(err) - - -errors = (FailedDirectory, GenericBuildError) diff --git a/pkgcore/interfaces/observer.py b/pkgcore/interfaces/observer.py deleted file mode 100644 index a98377e..0000000 --- a/pkgcore/interfaces/observer.py +++ /dev/null @@ -1,106 +0,0 @@ -# Copyright: 2006-2007 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -from snakeoil.currying import pre_curry - -class base(object): - - def warn(self, msg): - pass - - -class phase_observer(object): - - def phase_start(self, phase): - pass - - def phase_end(self, phase, status): - pass - - -class file_phase_observer(phase_observer): - - def __init__(self, out, semiquiet=True): - self._out = out - self._semiquiet = semiquiet - - def phase_start(self, phase): - if not self._semiquiet: - self._out.write("starting %s\n" % phase) - - def info(self, msg): - if not self._semiquiet: - self._out.write("info: %s\n" % msg) - - def warn(self, msg): - self._out.write("warning: %s\n" % msg) - - def phase_end(self, phase, status): - if not self._semiquiet: - self._out.write("finished %s: %s\n" % (phase, status)) - - -class build_observer(base, phase_observer): - pass - - -class repo_base(base): - pass - - -class repo_observer(repo_base, phase_observer): - - def trigger_start(self, hook, trigger): - pass - - trigger_end = trigger_start - - def installing_fs_obj(self, obj): - pass - - removing_fs_obj = installing_fs_obj - - -class file_build_observer(build_observer, file_phase_observer): - pass - - -class file_repo_observer(file_phase_observer, repo_base): - - def __init__(self, out, semiquiet=True): - self._out = out - self._semiquiet = semiquiet - - def trigger_start(self, hook, trigger): - if not self._semiquiet: - self._out.write("hook %s: trigger: starting %r\n" % (hook, trigger)) - - def trigger_end(self, hook, trigger): - if not self._semiquiet: - self._out.write("hook %s: trigger: finished %r\n" % (hook, trigger)) - - def installing_fs_obj(self, obj): - self._out.write(">>> %s\n" % obj) - - def removing_fs_obj(self, obj): - self._out.write("<<< %s\n" % obj) - - -def wrap_build_method(phase, method, self, *args, **kwds): - disable_observer = kwds.pop("disable_observer", False) - if self.observer is None or disable_observer: - return method(self, *args, **kwds) - self.observer.phase_start(phase) - ret = False - try: - ret = method(self, *args, **kwds) - finally: - self.observer.phase_end(phase, ret) - return ret - -def decorate_build_method(phase): - def f(func): - return pre_curry(wrap_build_method, phase, func) - return f - - diff --git a/pkgcore/interfaces/repo.py b/pkgcore/interfaces/repo.py deleted file mode 100644 index ac7c3cc..0000000 --- a/pkgcore/interfaces/repo.py +++ /dev/null @@ -1,331 +0,0 @@ -# Copyright: 2005-2007 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -""" -repository modifications (installing, removing, replacing) -""" - -from pkgcore.merge import errors as merge_errors -from pkgcore.merge.engine import MergeEngine -from snakeoil.dependant_methods import ForcedDepends -from snakeoil.demandload import demandload -demandload(globals(), "pkgcore.log:logger") - - -class fake_lock(object): - def __init__(self): - pass - - acquire_write_lock = acquire_read_lock = __init__ - release_read_lock = release_write_lock = __init__ - - -class base(object): - __metaclass__ = ForcedDepends - - stage_depends = {} - -class Failure(Exception): - pass - - -class nonlivefs_base(base): - - stage_depends = {'finish': '_notify_repo', '_notify_repo': 'modify_repo', - 'modify_repo':'start'} - - def __init__(self, repo, observer=None): - self.repo = repo - self.underway = False - self.observer = observer - self.lock = getattr(repo, "lock") - if self.lock is None: - self.lock = fake_lock() - - def start(self): - self.underway = True - self.lock.acquire_write_lock() - return True - - def modify_repo(self): - raise NotImplementedError(self, 'modify_repo') - - def _notify_repo(self): - raise NotImplementedError(self, '_notify_repo') - - def finish(self): - self._notify_repo() - self.lock.release_write_lock() - self.underway = False - return True - - -class nonlivefs_install(nonlivefs_base): - - def __init__(self, repo, pkg, **kwds): - nonlivefs_base.__init__(self, repo, **kwds) - self.new_pkg = pkg - - def _notify_repo(self): - self.repo.notify_add_package(self.new_pkg) - - -class nonlivefs_uninstall(nonlivefs_base): - - def __init__(self, repo, pkg, **kwds): - nonlivefs_base.__init__(self, repo, **kwds) - self.old_pkg = pkg - - def _notify_repo(self): - self.repo.notify_remove_package(self.old_pkg) - - -class nonlivefs_replace(nonlivefs_install, nonlivefs_uninstall): - - def __init__(self, repo, oldpkg, newpkg, **kwds): - # yes there is duplicate initialization here. - nonlivefs_uninstall.__init__(self, repo, oldpkg, **kwds) - nonlivefs_install.__init__(self, repo, newpkg, **kwds) - - def _notify_repo(self): - nonlivefs_uninstall._notify_repo(self) - nonlivefs_install._notify_repo(self) - - -class livefs_base(base): - stage_hooks = [] - - def __init__(self, repo, observer=None, offset=None): - self.repo = repo - self.underway = False - self.offset = offset - self.observer = observer - self.get_op() - self.lock = getattr(repo, "lock") - if self.lock is None: - self.lock = fake_lock() - - def customize_engine(self, engine): - pass - - def _get_format_op_args_kwds(self): - return (), {} - - def start(self, engine): - self.me = engine - self.underway = True - self.lock.acquire_write_lock() - self.me.sanity_check() - return True - - def finish(self): - """finish the transaction""" - self.me.final() - self._notify_repo() - self.lock.release_write_lock() - self.underway = False - return True - - def _modify_repo_cache(self): - raise NotImplementedError - - def __del__(self): - if self.underway: - print "warning: %s merge was underway, but wasn't completed" - self.lock.release_write_lock() - - -class livefs_install(livefs_base): - - """base interface for installing a pkg into a livefs repo. - - repositories should override as needed. - """ - - stage_depends = { - "finish":"merge_metadata", "merge_metadata":"postinst", - "postinst":"transfer", "transfer":"preinst", "preinst":"start"} - stage_hooks = ["merge_metadata", "postinst", "preinst", "transfer"] - install_op_name = "_repo_install_op" - - def __init__(self, repo, pkg, *args, **kwds): - self.new_pkg = pkg - livefs_base.__init__(self, repo, *args, **kwds) - - install_get_format_op_args_kwds = livefs_base._get_format_op_args_kwds - - def get_op(self): - op_args, op_kwds = self.install_get_format_op_args_kwds() - op_kwds["observer"] = self.observer - self.install_op = getattr(self.new_pkg, - self.install_op_name)(*op_args, **op_kwds) - - def start(self): - """start the install transaction""" - engine = MergeEngine.install(self.new_pkg, offset=self.offset, - observer=self.observer) - self.new_pkg.add_format_triggers(self, self.install_op, engine) - self.customize_engine(engine) - return livefs_base.start(self, engine) - - def preinst(self): - """execute any pre-transfer steps required""" - return self.install_op.preinst() - - def transfer(self): - """execute the actual transfer""" - for x in (self.me.pre_merge, self.me.merge, self.me.post_merge): - try: - x() - except merge_errors.NonFatalModification, e: - print "warning caught: %s" % e - return True - - def _notify_repo(self): - self.repo.notify_add_package(self.new_pkg) - - def postinst(self): - """execute any post-transfer steps required""" - return self.install_op.postinst() - - def merge_metadata(self): - """merge pkg metadata to the repository. Must be overrided""" - raise NotImplementedError - - def finish(self): - ret = self.install_op.finalize() - if not ret: - logger.warn("ignoring unexpected result from install finalize- " - "%r" % ret) - return livefs_base.finish(self) - - -class livefs_uninstall(livefs_base): - - """base interface for uninstalling a pkg from a livefs repo. - - Repositories should override as needed. - """ - - stage_depends = { - "finish":"unmerge_metadata", "unmerge_metadata":"postrm", - "postrm":"remove", "remove":"prerm", "prerm":"start"} - stage_hooks = ["merge_metadata", "postrm", "prerm", "remove"] - uninstall_op_name = "_repo_uninstall_op" - - def __init__(self, repo, pkg, *args, **kwds): - self.old_pkg = pkg - livefs_base.__init__(self, repo, *args, **kwds) - - uninstall_get_format_op_args_kwds = livefs_base._get_format_op_args_kwds - - def get_op(self): - op_args, op_kwds = self.uninstall_get_format_op_args_kwds() - op_kwds["observer"] = self.observer - self.uninstall_op = getattr(self.old_pkg, - self.uninstall_op_name)(*op_args, **op_kwds) - - def start(self): - """start the uninstall transaction""" - engine = MergeEngine.uninstall(self.old_pkg, offset=self.offset, - observer=self.observer) - self.old_pkg.add_format_triggers(self, self.uninstall_op, engine) - self.customize_engine(engine) - return livefs_base.start(self, engine) - - def prerm(self): - """execute any pre-removal steps required""" - return self.uninstall_op.prerm() - - def remove(self): - """execute any removal steps required""" - for x in (self.me.pre_unmerge, self.me.unmerge, self.me.post_unmerge): - try: - x() - except merge_errors.NonFatalModification, e: - print "warning caught: %s" % e - return True - - def postrm(self): - """execute any post-removal steps required""" - return self.uninstall_op.postrm() - - def _notify_repo(self): - self.repo.notify_remove_package(self.old_pkg) - - def unmerge_metadata(self): - """unmerge pkg metadata from the repository. Must be overrided.""" - raise NotImplementedError - - def finish(self): - ret = self.uninstall_op.finalize() - self.uninstall_op.cleanup(disable_observer=True) - if not ret: - logger.warn("ignoring unexpected result from uninstall finalize- " - "%r" % ret) - return livefs_base.finish(self) - - def __del__(self): - if self.underway: - print "warning: %s unmerge was underway, but wasn't completed" % \ - self.old_pkg - self.lock.release_write_lock() - - -class livefs_replace(livefs_install, livefs_uninstall): - - """base interface for replacing a pkg in a livefs repo with another. - - Repositories should override as needed. - """ - - stage_depends = { - "finish":"postinst", "postinst":"unmerge_metadata", - "unmerge_metadata":"postrm", "postrm":"remove", - "remove":"prerm", "prerm":"merge_metadata", - "merge_metadata":"transfer", - "transfer":"preinst", "preinst":"start"} - - stage_hooks = [ - "merge_metadata", "unmerge_metadata", "postrm", "prerm", "postinst", - "preinst", "unmerge_metadata", "merge_metadata"] - - def __init__(self, repo, oldpkg, newpkg, **kwds): - self.old_pkg = oldpkg - self.new_pkg = newpkg - livefs_base.__init__(self, repo, **kwds) - - def get_op(self): - livefs_install.get_op(self) - livefs_uninstall.get_op(self) - - def start(self): - """start the transaction""" - engine = MergeEngine.replace(self.old_pkg, self.new_pkg, - offset=self.offset, observer=self.observer) - self.old_pkg.add_format_triggers(self, self.uninstall_op, engine) - self.new_pkg.add_format_triggers(self, self.install_op, engine) - self.customize_engine(engine) - return livefs_base.start(self, engine) - - def _notify_repo(self): - self.repo.notify_remove_package(self.old_pkg) - self.repo.notify_add_package(self.new_pkg) - - def finish(self): - ret = self.install_op.finalize() - if not ret: - logger.warn("ignoring unexpected result from install finalize- " - "%r" % ret) - ret = self.uninstall_op.finalize() - self.uninstall_op.cleanup(disable_observer=True) - if not ret: - logger.warn("ignoring unexpected result from uninstall finalize- " - "%r" % ret) - return livefs_base.finish(self) - - def __del__(self): - if self.underway: - print "warning: %s -> %s replacement was underway, " \ - "but wasn't completed" % (self.old_pkg, self.new_pkg) - self.lock.release_write_lock() diff --git a/pkgcore/log.py b/pkgcore/log.py deleted file mode 100644 index 5035430..0000000 --- a/pkgcore/log.py +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright: 2006 Marien Zwart <marienz@gentoo.org> -# License: GPL2 - - -"""Logging utilities. - -Currently just contains pkgcore's root logger. -""" - - -import logging - -# The logging system will call this automagically if its module-level -# logging functions are used. We call it explicitly to make sure -# something handles messages sent to our non-root logger. If the root -# logger already has handlers this is a noop, and if someone attaches -# a handler to our pkgcore logger that overrides the root logger handler. -logging.basicConfig() - -# Our main logger. -logger = logging.getLogger('pkgcore') diff --git a/pkgcore/merge/__init__.py b/pkgcore/merge/__init__.py deleted file mode 100644 index b7fbd75..0000000 --- a/pkgcore/merge/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright: 2006 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -""" -package related livefs modification subsystem -""" diff --git a/pkgcore/merge/const.py b/pkgcore/merge/const.py deleted file mode 100644 index 7ccb140..0000000 --- a/pkgcore/merge/const.py +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright: 2006 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -REPLACE_MODE = 0 -INSTALL_MODE = 1 -UNINSTALL_MODE = 2 diff --git a/pkgcore/merge/engine.py b/pkgcore/merge/engine.py deleted file mode 100644 index 60e261d..0000000 --- a/pkgcore/merge/engine.py +++ /dev/null @@ -1,316 +0,0 @@ -# Copyright: 2005-2006 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -""" -core engine for livefs modifications -""" - -# need better documentation... - -# pre merge triggers -# post merge triggers -# ordering? - -import operator - -from pkgcore.fs import contents, livefs -from pkgcore.plugin import get_plugins -from pkgcore.merge import errors -from pkgcore.interfaces import observer as observer_mod -from pkgcore.merge.const import REPLACE_MODE, INSTALL_MODE, UNINSTALL_MODE - -from snakeoil.mappings import LazyValDict, ImmutableDict, StackedDict -from snakeoil import currying - -from snakeoil.demandload import demandload -demandload(globals(), 'errno') - -def alias_cset(alias, engine, csets): - """alias a cset to another""" - return csets[alias] - - -class MergeEngine(object): - - install_hooks = dict((x, []) for x in [ - "sanity_check", "pre_merge", "merge", "post_merge", "final"]) - uninstall_hooks = dict((x, []) for x in [ - "sanity_check", "pre_unmerge", "unmerge", "post_unmerge", "final"]) - replace_hooks = dict((x, []) for x in set( - install_hooks.keys() + uninstall_hooks.keys())) - - install_csets = {"install_existing":"get_install_livefs_intersect"} - uninstall_csets = { - "uninstall_existing":"get_uninstall_livefs_intersect", - "uninstall":currying.partial(alias_cset, "old_cset")} - replace_csets = dict(install_csets) - replace_csets.update(uninstall_csets) - - install_csets.update({}.fromkeys(["install", "replace"], - currying.partial(alias_cset, "new_cset"))) - replace_csets["install"] = currying.partial(alias_cset, "new_cset") - replace_csets["modifying"] = ( - lambda e, c: c["install"].intersection(c["uninstall"])) - replace_csets["uninstall"] = "get_remove_cset" - replace_csets["replace"] = "get_replace_cset" - replace_csets["install_existing"] = "get_install_livefs_intersect" - - install_csets_preserve = ["new_cset"] - uninstall_csets_preserve = ["old_cset"] - replace_csets_preserve = ["new_cset", "old_cset"] - - - def __init__(self, mode, hooks, csets, preserves, observer, offset=None): - if observer is None: - observer = observer_mod.repo_observer() - self.observer = observer - self.mode = mode - - self.hooks = ImmutableDict((x, []) for x in hooks) - - self.preserve_csets = [] - self.cset_sources = {} - # instantiate these seperately so their values are preserved - self.preserved_csets = LazyValDict( - self.preserve_csets, self._get_cset_source) - for k, v in csets.iteritems(): - if isinstance(v, basestring): - v = getattr(self, v, v) - if not callable(v): - raise TypeError( - "cset values must be either the string name of " - "existing methods, or callables (got %s)" % v) - - if k in preserves: - self.add_preserved_cset(k, v) - else: - self.add_cset(k, v) - - if offset is None: - offset = "/" - self.offset = offset - - # merge in default triggers first. - for trigger in get_plugins('triggers'): - t = trigger() - t.register(self) - - # merge in overrides - for hook, triggers in hooks.iteritems(): - for trigger in triggers: - self.add_trigger(hook, trigger) - - self.regenerate_csets() - for x in hooks.keys(): - setattr(self, x, currying.partial(self.execute_hook, x)) - - @classmethod - def install(cls, pkg, offset=None, observer=None): - - """ - generate a MergeEngine instance configured for uninstalling a pkg - - @param pkg: L{pkgcore.package.metadata.package} instance to install - @param offset: any livefs offset to force for modifications - @return: L{MergeEngine} - - """ - - hooks = dict( - (k, [y() for y in v]) - for (k, v) in cls.install_hooks.iteritems()) - - csets = dict(cls.install_csets) - if "new_cset" not in csets: - csets["new_cset"] = currying.post_curry(cls.get_pkg_contents, pkg) - o = cls( - INSTALL_MODE, hooks, csets, cls.install_csets_preserve, - observer, offset=offset) - - if o.offset != '/': - # wrap the results of new_cset to pass through an offset generator - o.cset_sources["new_cset"] = currying.post_curry( - o.generate_offset_cset, o.cset_sources["new_cset"]) - - o.new = pkg - return o - - @classmethod - def uninstall(cls, pkg, offset=None, observer=None): - - """ - generate a MergeEngine instance configured for uninstalling a pkg - - @param pkg: L{pkgcore.package.metadata.package} instance to uninstall, - must be from a livefs vdb - @param offset: any livefs offset to force for modifications - @return: L{MergeEngine} - """ - - hooks = dict( - (k, [y() for y in v]) - for (k, v) in cls.uninstall_hooks.iteritems()) - csets = dict(cls.uninstall_csets) - if "old_cset" not in csets: - csets["old_cset"] = currying.post_curry(cls.get_pkg_contents, pkg) - o = cls( - UNINSTALL_MODE, hooks, csets, cls.uninstall_csets_preserve, - observer, offset=offset) - - if o.offset != '/': - # wrap the results of new_cset to pass through an offset generator - o.cset_sources["old_cset"] = currying.post_curry( - o.generate_offset_cset, o.cset_sources["old_cset"]) - - o.old = pkg - return o - - @classmethod - def replace(cls, old, new, offset=None, observer=None): - - """ - generate a MergeEngine instance configured for replacing a pkg. - - @param old: L{pkgcore.package.metadata.package} instance to replace, - must be from a livefs vdb - @param new: L{pkgcore.package.metadata.package} instance - @param offset: any livefs offset to force for modifications - @return: L{MergeEngine} - - """ - - hooks = dict( - (k, [y() for y in v]) - for (k, v) in cls.replace_hooks.iteritems()) - - csets = dict(cls.replace_csets) - - for v, k in ((old, "old_cset"), (new, "new_cset")): - if k not in csets: - csets[k] = currying.post_curry(cls.get_pkg_contents, v) - - o = cls( - REPLACE_MODE, hooks, csets, cls.replace_csets_preserve, - observer, offset=offset) - - if o.offset != '/': - for k in ("old_cset", "new_cset"): - # wrap the results of new_cset to pass through an - # offset generator - o.cset_sources[k] = currying.post_curry( - o.generate_offset_cset, o.cset_sources[k]) - - o.old = old - o.new = new - return o - - def regenerate_csets(self): - """ - internal function, reset non preserverd csets. - - Used in transitioning between hook points - """ - self.csets = StackedDict(self.preserved_csets, - LazyValDict(self.cset_sources, self._get_cset_source)) - - def _get_cset_source(self, key): - return self.cset_sources[key](self, self.csets) - - def add_preserved_cset(self, cset_name, func): - """ - register a cset generator for use. - - The cset will stay in memory until the engine finishes all steps. - - @param cset_name: what to call the generated cset - @param func: callable to get the cset - """ - self.add_cset(cset_name, func) - self.preserve_csets.append(cset_name) - - def add_cset(self, cset_name, func): - """ - regiser a cset generator for use. - - The cset will be released from memory when it's no longer used. - - @param cset_name: what to call the generated cset - @param func: callable to get the cset - """ - if not callable(func): - raise TypeError("func must be a callable") - if not isinstance(cset_name, basestring): - raise TypeError("cset_name must be a string") - self.cset_sources[cset_name] = func - - def add_trigger(self, hook_name, trigger, required_csets): - """ - register a L{pkgcore.merge.triggers.base} instance to be executed - - @param hook_name: engine step to hook the trigger into - @param trigger: L{triggers<pkgcore.merge.triggers.base>} to add - """ - if hook_name not in self.hooks: - raise KeyError("trigger %r's hook %s isn't a known hook" % - (trigger, hook_name)) - - if required_csets is not None: - for rcs in required_csets: - if rcs not in self.cset_sources: - if isinstance(rcs, basestring): - raise errors.TriggerUnknownCset(trigger, rcs) - - self.hooks[hook_name].append(trigger) - - def execute_hook(self, hook): - """ - execute any triggers bound to a hook point - """ - try: - self.phase = hook - self.regenerate_csets() - for trigger in sorted(self.hooks[hook], - key=operator.attrgetter("priority")): - # error checking needed here. - self.observer.trigger_start(hook, trigger) - try: - trigger(self, self.csets) - finally: - self.observer.trigger_end(hook, trigger) - finally: - self.phase = None - - @staticmethod - def generate_offset_cset(engine, csets, cset_generator): - """generate a cset with offset applied""" - return cset_generator(engine, csets).insert_offset(engine.offset) - - @staticmethod - def get_pkg_contents(engine, csets, pkg): - """generate the cset of what files shall be merged to the livefs""" - return pkg.contents.clone() - - @staticmethod - def get_remove_cset(engine, csets): - """generate the cset of what files shall be removed from the livefs""" - return csets["old_cset"].difference(csets["new_cset"]) - - @staticmethod - def get_replace_cset(engine, csets): - """Return the cset of what will be replaced going from old->new pkg.""" - return csets["new_cset"].intersection(csets["old_cset"]) - - @staticmethod - def _get_livefs_intersect_cset(engine, csets, cset_name): - """generates the livefs intersection against a cset""" - return contents.contentsSet(livefs.intersect(csets[cset_name])) - - @staticmethod - def get_install_livefs_intersect(engine, csets): - return engine._get_livefs_intersect_cset(engine, csets, "install") - - @staticmethod - def get_uninstall_livefs_intersect(engine, csets): - return engine._get_livefs_intersect_cset(engine, csets, "uninstall") - - alias_cset = staticmethod(alias_cset) diff --git a/pkgcore/merge/errors.py b/pkgcore/merge/errors.py deleted file mode 100644 index b489f24..0000000 --- a/pkgcore/merge/errors.py +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright: 2006 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -""" -exceptions thrown by the MergeEngine -""" - -class ModificationError(Exception): - - """Base Exception class for modification errors/warnings""" - - def __init__(self, trigger, msg): - self.trigger = trigger - self.msg = msg - Exception.__init__(self, "%s: modification error: %s" % - (self.trigger, self.msg)) - - -class BlockModification(ModificationError): - """Merging cannot proceed""" - - def __str__(self): - return "Modification was blocked by %s: %s" % ( - self.trigger.__class__.__name__, self.msg) - -class TriggerUnknownCset(ModificationError): - """Trigger's required content set isn't known""" - - def __init__(self, trigger, csets): - if not isinstance(csets, (tuple, list)): - csets = (csets,) - ModificationError.__init__(self, "%s: trigger %r unknown cset: %r" % - (self.__class__, trigger, csets)) - self.trigger, self.csets = trigger, csets - - -class NonFatalModification(Exception): - pass - -class TriggerWarning(NonFatalModification): - pass - diff --git a/pkgcore/merge/todo.txt b/pkgcore/merge/todo.txt deleted file mode 100644 index 023c7ec..0000000 --- a/pkgcore/merge/todo.txt +++ /dev/null @@ -1,40 +0,0 @@ -missing triggers: -implement INSTALL_MASK - - -- misc-functions.sh - 1) big ass scanelf block. - 2) install_mask (need to bind domain generation of triggers for it) - 3) preinst_mask (same thing, although that shouldn't wipe mangle the install image) - 4) sfperms (feature based), domain bound - 5) suid control. same thing (see a pattern?) - 6) selinux labelling. need to override the copies there imo, installing then slapping labels on sucks, although could mangle the image file and use a selinux aware copy - -prepman: - 1) all of it. the symlink rewriting might be fun... - -prepinfo: - 1) no different then prepman. - -prepstrip: - 1) splitdebug (transformation, fun one that one- maybe higher up, generate N pkgs instead) - 2) installsources - - -prepall: (calls prepman, prepinfo, and prepstrip which are seperated in this list) - 1) qa: bug 4111, gen_usr_ldscript shit for static files. - 2) qa: check for la/.a in /lib - 3) more scanelf idiocy- check for libs without sonames, no NEEDED info. - -not automatically invoked- - -prepalldocs: - 1) symlink/compression. usual. - -potential -preplib: - 1) we can generate this ourselves... figure out if ebuilds really should be doing it themselves (eapi bump for that most likely) - -size check (is there enough space on the partitions for merging?) -revdep check. - diff --git a/pkgcore/merge/triggers.py b/pkgcore/merge/triggers.py deleted file mode 100644 index a0fd43f..0000000 --- a/pkgcore/merge/triggers.py +++ /dev/null @@ -1,539 +0,0 @@ -# Copyright: 2006-2007 Brian Harring <ferringb@gmail.com> -# License: GPL2 -# $Id:$ - -""" -triggers, callables to bind to a step in a MergeEngine to affect changes -""" - -__all__ = [ - "base", - "trigger", - "UNINSTALLING_MODES", - "INSTALLING_MODES" -] - -from pkgcore.merge import errors, const -import pkgcore.os_data - -from snakeoil.osutils import listdir_files, pjoin, ensure_dirs, normpath -from snakeoil.demandload import demandload -demandload(globals(), - 'os', - 'errno', - 'pkgcore.plugin:get_plugin', - 'pkgcore:spawn', - 'pkgcore.fs.livefs:gen_obj', - 'pkgcore.fs:fs,contents', - 'snakeoil.fileutils:iter_read_bash', - 'time', - 'math:floor', -) - -UNINSTALLING_MODES = (const.REPLACE_MODE, const.UNINSTALL_MODE) -INSTALLING_MODES = (const.REPLACE_MODE, const.INSTALL_MODE) - - -class base(object): - - """base trigger class - - @ivar required_csets: If None, all csets are passed in, else it must be a - sequence, those specific csets are passed in - @ivar _label: Either None, or a string to use for this triggers label - @ivar _hook: sequence of hook points to register into - @ivar _priority: range of 0 to 100, order of execution for triggers per hook - @ivar _engine_types: if None, trigger works for all engine modes, else it's - limited to that mode, and must be a sequence - """ - - required_csets = None - _label = None - _hooks = None - _engine_types = None - _priority = 50 - - @property - def priority(self): - return self._priority - - @property - def label(self): - if self._label is not None: - return self._label - return str(self.__class__.__name__) - - def register(self, engine): - """ - register with a MergeEngine - """ - if self._engine_types is not None and \ - engine.mode not in self._engine_types: - return - - # ok... so we care about this mode. - try: - i = iter(self._hooks) - except TypeError: - # bad monkey... - raise TypeError("%r: %r: _hooks needs to be a sequence" % - (self, self._hooks)) - - csets = self.get_required_csets(engine.mode) - - for hook in self._hooks: - try: - engine.add_trigger(hook, self, csets) - except KeyError: - # unknown hook. - continue - - def get_required_csets(self, mode): - csets = self.required_csets - if csets is not None: - if not isinstance(csets, tuple): - # has to be a dict. - csets = csets.get(mode) - return csets - - def localize(self, mergeengine): - """ - 'localize' a trigger to a specific merge engine process - mainly used if the trigger comes from configuration - """ - return self - - @staticmethod - def _get_csets(required_csets, csets): - return [csets[x] for x in required_csets] - - def trigger(self, engine, csets): - raise NotImplementedError(self, 'trigger') - - def __call__(self, engine, csets): - """execute the trigger""" - - required_csets = self.get_required_csets(engine.mode) - - if required_csets is None: - return self.trigger(engine, csets) - return self.trigger(engine, *self._get_csets(required_csets, csets)) - - def __str__(self): - return "%s: cset(%s) ftrigger(%s)" % ( - self.label, self.required_csets, self.trigger) - - def __repr__(self): - return "<%s cset=%r @#%x>" % ( - self.label, - self.required_csets, id(self)) - - -class mtime_watcher(object): - """ - passed a list of locations, return a L{contents.contentsSet} containing - those that are directories. - - If the location doesn't exist, it's ignored. If stat_func is os.stat - and the location is a symlink pointing at a non existant location, it's - ignored. - - Additionally, since this function is used for effectively 'snapshotting' - related directories, if any mtimes are *now* (fs doesn't do subsecond - resolution, osx for example), induces a sleep for a second to ensure - any later re-runs do not get bit by completing within the race window. - - Finally, if any mtime is detected that is in the future, it is reset - to 'now'. - """ - - def __init__(self): - self.saved_mtimes = None - self.locations = None - - def mtime_floats(func): - def mtime_floats_wrapper(self, *args, **kwargs): - cur = os.stat_float_times() - try: - os.stat_float_times(True) - return func(self, *args, **kwargs) - finally: - os.stat_float_times(cur) - return mtime_floats_wrapper - - def __nonzero__(self): - return bool(self.saved_mtimes) - - @staticmethod - def _scan_mtimes(locations, stat_func): - for x in locations: - try: - st = stat_func(x) - except OSError, oe: - if not oe.errno == errno.ENOENT: - raise - continue - obj = gen_obj(x, stat=st) - if fs.isdir(obj): - yield obj - - @mtime_floats - def set_state(self, locations, stat_func=os.stat, forced_past=2): - """ - set the initial state; will adjust ondisk mtimes as needed - to avoid race potentials. - - @param locations: sequence, file paths to scan - @param stat_func: stat'er to use. defaults to os.stat - """ - self.locations = locations - mtimes = list(self._scan_mtimes(locations, stat_func)) - - cset = contents.contentsSet(mtimes) - now = time.time() - pause_cutoff = floor(now) - past = max(pause_cutoff - forced_past, 0) - resets = [x for x in mtimes if x.mtime > past] - for x in resets: - cset.add(x.change_attributes(mtime=past)) - os.utime(x.location, (past, past)) - - self.saved_mtimes = cset - - @mtime_floats - def check_state(self, locations=None, stat_func=os.stat): - """ - set the initial state; will adjust ondisk mtimes as needed - to avoid race potentials. - - @param locations: sequence, file paths to scan; uses the locations - from the set_state invocation if not supplised. - @param stat_func: stat'er to use. defaults to os.stat - @return: boolean, True if things have changed, False if not. - """ - if locations is None: - locations = self.locations - - for x in self.get_changes(locations=locations, stat_func=stat_func): - return True - return False - - @mtime_floats - def get_changes(self, locations=None, stat_func=os.stat): - """ - generator yielding the fs objs for what has changed. - - @param locations: sequence, file paths to scan; uses the locations - from the set_state invocation if not supplised. - @param stat_func: stat'er to use. defaults to os.stat - """ - if locations is None: - locations = self.locations - - for x in self._scan_mtimes(locations, stat_func): - if x not in self.saved_mtimes or \ - self.saved_mtimes[x].mtime != x.mtime: - yield x - - -class ldconfig(base): - - required_csets = () - _engine_types = None - _hooks = ('pre_merge', 'post_merge', 'pre_unmerge', 'post_unmerge') - _priority = 10 - - default_ld_path = ['usr/lib', 'usr/lib64', 'usr/lib32', 'lib', - 'lib64', 'lib32'] - - def __init__(self, ld_so_conf_path="etc/ld.so.conf"): - self.ld_so_conf_path = ld_so_conf_path.lstrip(os.path.sep) - self.saved_mtimes = mtime_watcher() - - def ld_so_path(self, offset): - return pjoin(offset, self.ld_so_conf_path) - - def read_ld_so_conf(self, offset): - fp = self.ld_so_path(offset) - - try: - l = [x.lstrip(os.path.sep) for x in iter_read_bash(fp)] - except IOError, oe: - if oe.errno != errno.ENOENT: - raise - self._mk_ld_so_conf(fp) - # fall back to an edjucated guess. - l = self.default_ld_path - return [pjoin(offset, x) for x in l] - - def _mk_ld_so_conf(self, fp): - if not ensure_dirs(os.path.dirname(fp), mode=0755, minimal=True): - raise errors.BlockModification(self, - "failed creating/setting %s to 0755, root/root for uid/gid" % - os.path.basename(fp)) - # touch the file. - try: - open(fp, 'w') - except (IOError, OSError), e: - raise errors.BlockModification(self, e) - - def trigger(self, engine): - locations = self.read_ld_so_conf(engine.offset) - if engine.phase.startswith('pre_'): - self.saved_mtimes.set_state(locations) - return - - if self.saved_mtimes.check_state(locations): - self.regen(engine.offset) - - def regen(self, offset): - ret = spawn.spawn(["/sbin/ldconfig", "-r", offset], fd_pipes={1:1, 2:2}) - if ret != 0: - raise errors.TriggerWarning(self, - "ldconfig returned %i from execution" % ret) - - -class InfoRegen(base): - - required_csets = () - - # could implement this to look at csets, and do incremental removal and - # addition; doesn't seem worth while though for the additional complexity - - _hooks = ('pre_merge', 'post_merge', 'pre_unmerge', 'post_unmerge') - _engine_types = None - _label = "gnu info regen" - - locations = ('/usr/share/info',) - - def __init__(self): - self.saved_mtimes = mtime_watcher() - - def get_binary_path(self): - try: - return spawn.find_binary('install-info') - except spawn.CommandNotFound: - # swallow it. - return None - - def trigger(self, engine): - bin_path = self.get_binary_path() - if bin_path is None: - return - - offset = engine.offset - - locs = [pjoin(offset, x.lstrip(os.path.sep)) for x in self.locations] - - if engine.phase.startswith('pre_'): - self.saved_mtimes.set_state(locs) - return - elif engine.phase == 'post_merge' and \ - engine.mode == const.REPLACE_MODE: - # skip post_merge for replace. - # we catch it on unmerge... - return - - regens = set(x.location for x in self.saved_mtimes.get_changes(locs)) - # force regeneration of any directory lacking the info index. - regens.update(x for x in locs if not os.path.isfile(pjoin(x, 'dir'))) - - bad = [] - for x in regens: - bad.extend(self.regen(bin_path, x)) - - if bad and engine.observer is not None: - engine.observer.warn("bad info files: %r" % sorted(bad)) - - def regen(self, binary, basepath): - ignores = ("dir", "dir.old") - try: - files = listdir_files(basepath) - except OSError, oe: - if oe.errno == errno.ENOENT: - return - raise - - # wipe old indexes. - for x in set(ignores).intersection(files): - os.remove(pjoin(basepath, x)) - - index = pjoin(basepath, 'dir') - for x in files: - if x in ignores or x.startswith("."): - continue - - ret, data = spawn.spawn_get_output( - [binary, '--quiet', pjoin(basepath, x), - '--dir-file', index], - collect_fds=(1,2), split_lines=False) - - if not data or "already exists" in data or \ - "warning: no info dir entry" in data: - continue - yield pjoin(basepath, x) - - -class merge(base): - - required_csets = ('install',) - _engine_types = INSTALLING_MODES - _hooks = ('merge',) - - def trigger(self, engine, merging_cset): - op = get_plugin('fs_ops.merge_contents') - return op(merging_cset, callback=engine.observer.installing_fs_obj) - - -class unmerge(base): - - required_csets = ('uninstall',) - _engine_types = UNINSTALLING_MODES - _hooks = ('unmerge',) - - def trigger(self, engine, unmerging_cset): - op = get_plugin('fs_ops.unmerge_contents') - return op(unmerging_cset, callback=engine.observer.removing_fs_obj) - - -class fix_uid_perms(base): - - required_csets = ('new_cset',) - _hooks = ('pre_merge',) - _engine_types = INSTALLING_MODES - - def __init__(self, uid=pkgcore.os_data.portage_uid, - replacement=pkgcore.os_data.root_uid): - - base.__init__(self) - self.bad_uid = uid - self.good_uid = replacement - - def trigger(self, engine, cset): - good = self.good_uid - bad = self.bad_uid - - cset.update(x.change_attributes(uid=good) - for x in cset if x.uid == bad) - - -class fix_gid_perms(base): - - required_csets = ('new_cset',) - _hooks = ('pre_merge',) - _engine_types = INSTALLING_MODES - - def __init__(self, gid=pkgcore.os_data.portage_gid, - replacement=pkgcore.os_data.root_gid): - - base.__init__(self) - self.bad_gid = gid - self.good_gid = replacement - - def trigger(self, engine, cset): - good = self.good_gid - bad = self.bad_gid - - cset.update(x.change_attributes(gid=good) - for x in cset if x.gid == bad) - - -class fix_set_bits(base): - - required_csets = ('new_cset',) - _hooks = ('pre_merge',) - _engine_types = INSTALLING_MODES - - def trigger(self, engine, cset): - reporter = engine.observer - # if s(uid|gid) *and* world writable... - l = [x for x in cset.iterlinks(True) if - (x.mode & 06000) and (x.mode & 0002)] - - if reporter is not None: - for x in l: - if x.mode & 04000: - reporter.warn( - "correcting unsafe world writable SetGID: %s" % - (x.location,)) - else: - reporter.warn( - "correcting unsafe world writable SetUID: %s" % - (x.location,)) - - if l: - # wipe setgid/setuid - cset.update(x.change_attributes(mode=x.mode & ~06002) for x in l) - - -class detect_world_writable(base): - - required_csets = ('new_cset',) - _hooks = ('pre_merge',) - _engine_types = INSTALLING_MODES - - def __init__(self, fix_perms=False): - base.__init__(self) - self.fix_perms = fix_perms - - def trigger(self, engine, cset): - if not engine.observer and not self.fix_perms: - return - - reporter = engine.observer - - l = [x for x in cset.iterlinks(True) if x.mode & 0002] - if reporter is not None: - for x in l: - reporter.warn("world writable file: %s" % x.location) - if self.fix_perms: - cset.update(x.change_attributes(mode=x.mode & ~0002) for x in l) - - -class PruneFiles(base): - - required_csets = ('new_cset',) - _hooks = ('pre_merge',) - _engine_types = INSTALLING_MODES - - def __init__(self, sentinel_func): - """ - @param sentinel_func: callable accepting a fsBase entry, returns - True if the entry should be removed, False otherwise - """ - base.__init__(self) - self.sentinel = sentinel_func - - def trigger(self, engine, cset): - removal = filter(self.sentinel, cset) - if engine.observer: - for x in removal: - engine.observer.info("pruning: %s" % x.location) - cset.difference_update(removal) - - -class CommonDirectoryModes(base): - - required_csets = ('new_cset',) - _hooks = ('pre_merge',) - _engine_types = INSTALLING_MODES - - directories = [pjoin('/usr', x) for x in ('.', 'lib', 'lib64', 'lib32', - 'bin', 'sbin', 'local')] - directories.extend(pjoin('/usr/share', x) for x in ('.', 'man', 'info')) - directories.extend('/usr/share/man/man%i' % x for x in xrange(1, 10)) - directories.extend(['/lib', '/lib32', '/lib64', '/etc', '/bin', '/sbin', - '/var']) - directories = frozenset(map(normpath, directories)) - del x - - def trigger(self, engine, cset): - r = engine.observer - if not r: - return - for x in cset.iterdirs(): - if x.location not in self.directories: - continue - if x.mode != 0755: - r.warn('%s path has mode %s, should be 0755' % - (x.location, oct(x.mode))) diff --git a/pkgcore/os_data.py b/pkgcore/os_data.py deleted file mode 100644 index b7ab71c..0000000 --- a/pkgcore/os_data.py +++ /dev/null @@ -1,74 +0,0 @@ -# Copyright: 2005 Gentoo Foundation -# License: GPL2 - -""" -Avoid using- os data- root uid/gid, pkgcore uid/gid, etc. - -This will be killed off and bound into configuration subsystem at some point -""" - -import os, pwd, grp - -ostype = os.uname()[0] - -if ostype == "Linux": - userland = "GNU" - xargs = os.environ["XARGS"] = "xargs -r" - lchown = os.lchown -elif ostype == "Darwin": - userland = "Darwin" - xargs = os.environ["XARGS"] = "xargs" - def lchown(*pos_args, **key_args): - pass -elif ostype in ["FreeBSD", "OpenBSD", "NetBSD"]: - userland = "BSD" - xargs = os.environ["XARGS"] = "xargs" - lchown = os.lchown -else: - raise Exception("Operating system unsupported, '%s'" % ostype) - - -#os.environ["USERLAND"] = userland - -#Secpass will be set to 1 if the user is root or in the portage group. -secpass = 0 - -uid = os.getuid() -# hard coding sucks. -root_uid = 0 -root_gid = wheelgid = 0 - -if uid == 0: - secpass = 2 -try: - wheelgid = grp.getgrnam("wheel").gr_gid - if (not secpass) and (wheelgid in os.getgroups()): - secpass = 1 -except KeyError: - print "portage initialization: your system doesn't have a 'wheel' group." - print ("Please fix this as it is a normal system requirement. " - "'wheel' is GID 10") - print "'emerge baselayout' and an 'etc-update' should remedy this problem." - -#Discover the uid and gid of the portage user/group -try: - portage_uid = pwd.getpwnam("portage").pw_uid - portage_gid = grp.getgrnam("portage").gr_gid - portage_user_groups = tuple(x.gr_name for x in grp.getgrall() - if 'portage' in x.gr_mem) - - if (secpass == 0): - secpass = 1 -except KeyError: - portage_uid = 0 - portage_gid = wheelgid - portage_user_groups = [] - print - print "'portage' user or group missing. Please update baselayout" - print "and merge portage user(250) and group(250) into your passwd" - print "and group files. Non-root compilation is disabled until then." - print "Also note that non-root/wheel users will need to be added to" - print "the portage group to do portage commands.\n" - print "For the defaults, line 1 goes into passwd, and 2 into group." - print "portage:x:250:250:portage:/var/tmp/portage:/bin/false" - print "portage::250:portage" diff --git a/pkgcore/package/__init__.py b/pkgcore/package/__init__.py deleted file mode 100644 index 73d322b..0000000 --- a/pkgcore/package/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -# Copyright: 2005 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -""" -package interface/classes -""" - -# cpv and atom circularly import each other. This enforces a working order. -#import cpv - diff --git a/pkgcore/package/base.py b/pkgcore/package/base.py deleted file mode 100644 index 00684e6..0000000 --- a/pkgcore/package/base.py +++ /dev/null @@ -1,67 +0,0 @@ -# Copyright: 2005-2007 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -""" -base package class; instances should derive from this. - -Right now, doesn't provide much, need to change that down the line -""" - -class base(object): - - built = False - configurable = False - - __slots__ = ("__weakref__",) - _get_attr = {} - - def __setattr__(self, attr, value): - raise AttributeError(self, attr) - - def __delattr__(self, attr): - raise AttributeError(self, attr) - - def __getattr__(self, attr): - try: - val = self._get_attr[attr](self) - object.__setattr__(self, attr, val) - return val - except KeyError: - raise AttributeError(self, attr) - - @property - def versioned_atom(self): - raise NotImplementedError(self, "versioned_atom") - - @property - def unversioned_atom(self): - raise NotImplementedError(self, "versioned_atom") - - -class wrapper(base): - - __slots__ = ("_raw_pkg",) - - def __init__(self, raw_pkg): - object.__setattr__(self, "_raw_pkg", raw_pkg) - - def __cmp__(self, other): - if isinstance(other, wrapper): - return cmp(self._raw_pkg, other._raw_pkg) - return cmp(self._raw_pkg, other) - - def __eq__(self, other): - if isinstance(other, wrapper): - return cmp(self._raw_pkg, other._raw_pkg) == 0 - return cmp(self._raw_pkg, other) == 0 - - def __ne__(self, other): - return not self == other - - @property - def versioned_atom(self): - return self.raw_pkg.versioned_atom - - @property - def unversioned_atom(self): - return self.raw_pkg.unversioned_atom diff --git a/pkgcore/package/conditionals.py b/pkgcore/package/conditionals.py deleted file mode 100644 index 96456b1..0000000 --- a/pkgcore/package/conditionals.py +++ /dev/null @@ -1,249 +0,0 @@ -# Copyright: 2005-2007 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -""" -conditional attributes on a package. - -Changing them triggering regen of other attributes on the package instance. -""" - -from operator import attrgetter -from pkgcore.package.base import wrapper - -from snakeoil.containers import LimitedChangeSet, Unchangable -from snakeoil.klass import GetAttrProxy -from snakeoil.currying import partial -from snakeoil.demandload import demandload -demandload(globals(), "copy") - - -def _getattr_wrapped(attr, self): - o = self._cached_wrapped.get(attr) - if o is None or o[0] != self._reuse_pt: - o = self._wrapped_attr[attr](getattr(self._raw_pkg, attr), - self._configurable) - o = self._cached_wrapped[attr] = (self._reuse_pt, o) - return o[1] - - -def make_wrapper(configurable_attribute_name, attributes_to_wrap=()): - """ - @param configurable_attribute_name: attribute name to add, - and that is used for evaluating attributes_to_wrap - @param attributes_to_wrap: mapping of attr_name:callable - for revaluating the pkg_instance, using the result - instead of the wrapped pkgs attr. - """ - - if configurable_attribute_name.find(".") != -1: - raise ValueError("can only wrap first level attributes, " - "'obj.dar' fex, not '%s'" % - (configurable_attribute_name)) - - class PackageWrapper(wrapper): - """Add a new attribute, and evaluate attributes of a wrapped pkg.""" - - __slots__ = ("_unchangable", "_configurable", - "_reuse_pt", "_cached_wrapped", "_buildable") - - _wrapped_attr = attributes_to_wrap - _configurable_name = configurable_attribute_name - - configurable = True - - locals()[configurable_attribute_name] = \ - property(attrgetter("_configurable")) - - locals().update((x, property(partial(_getattr_wrapped, x))) - for x in attributes_to_wrap) - - __getattr__ = GetAttrProxy("_raw_pkg") - - def __init__(self, pkg_instance, - initial_settings=None, unchangable_settings=None, - build_callback=None): - - """ - @type pkg_instance: L{pkgcore.package.metadata.package} - @param pkg_instance: instance to wrap. - @type initial_settings: sequence - @param initial_settings: initial configuration of the - configurable_attribute - @type unchangable_settings: sequence - @param unchangable_settings: settings that configurable_attribute - cannot be set to - @param build_callback: None, or a callable to be used to get a - L{pkgcore.interfaces.format.build_base} instance - """ - - if initial_settings is None: - initial_settings = [] - if unchangable_settings is None: - unchangable_settings = [] - - sf = object.__setattr__ - sf(self, '_unchangable', unchangable_settings) - sf(self, '_configurable', - LimitedChangeSet(initial_settings, unchangable_settings)) - sf(self, '_reuse_pt', 0) - sf(self, '_cached_wrapped', {}) - sf(self, '_buildable', build_callback) - wrapper.__init__(self, pkg_instance) - - def __copy__(self): - return self.__class__(self._raw_pkg, self._configurable_name, - initial_settings=set(self._configurable), - unchangable_settings=self._unchangable, - attributes_to_wrap=self._wrapped_attr) - - def rollback(self, point=0): - """ - rollback changes to the configurable attribute to an earlier point - - @param point: must be an int - """ - self._configurable.rollback(point) - # yes, nuking objs isn't necessarily required. easier this way though. - # XXX: optimization point - object.__setattr__(self, '_reuse_pt', self._reuse_pt + 1) - - def commit(self): - """ - Commit current changes. - - This means that those changes can be reverted from this point out. - """ - self._configurable.commit() - object.__setattr__(self, '_reuse_pt', 0) - - def changes_count(self): - """ - current commit point for the configurable - """ - return self._configurable.changes_count() - - def request_enable(self, attr, *vals): - """ - internal function - - since configurable somewhat steps outside of normal - restriction protocols, request_enable requests that this - package instance change its configuration to make the - restriction return True; if not possible, reverts any changes - it attempted - - @param attr: attr to try and change - @param vals: L{pkgcore.restrictions.values.base} instances that - we're attempting to make match True - """ - if attr not in self._wrapped_attr: - if attr == self._configurable_name: - entry_point = self.changes_count() - try: - map(self._configurable.add, vals) - object.__setattr__(self, '_reuse_pt', - self._reuse_pt + 1) - return True - except Unchangable: - self.rollback(entry_point) - else: - a = getattr(self._raw_pkg, attr) - for x in vals: - if x not in a: - break - else: - return True - return False - entry_point = self.changes_count() - a = getattr(self._raw_pkg, attr) - try: - for x in vals: - succeeded = False - for reqs in a.node_conds.get(x, ()): - succeeded = reqs.force_True(self) - if succeeded: - break - if not succeeded: - self.rollback(entry_point) - return False - except Unchangable: - self.rollback(entry_point) - return False - object.__setattr__(self, '_reuse_pt', self._reuse_pt + 1) - return True - - def request_disable(self, attr, *vals): - """ - internal function - - since configurable somewhat steps outside of normal - restriction protocols, request_disable requests that this - package instance change its configuration to make the - restriction return False; if not possible, reverts any changes - it attempted - - @param attr: attr to try and change - @param vals: L{pkgcore.restrictions.values.base} instances that - we're attempting to make match False - """ - if attr not in self._wrapped_attr: - if attr == self._configurable_name: - entry_point = self.changes_count() - try: - map(self._configurable.remove, vals) - return True - except Unchangable: - self.rollback(entry_point) - else: - a = getattr(self._raw_pkg, attr) - for x in vals: - if x in a: - break - else: - return True - return False - entry_point = self.changes_count() - a = getattr(self._raw_pkg, attr) - try: - for x in vals: - succeeded = False - for reqs in a.node_conds.get(x, ()): - succeeded = reqs.force_False(self) - if succeeded: - break - if not succeeded: - self.rollback(entry_point) - return False - except Unchangable: - self.rollback(entry_point) - return False - object.__setattr__(self, '_reuse_pt', self._reuse_pt + 1) - return True - - def __str__(self): - return "config wrapped(%s): %s" % (self._configurable_name, - self._raw_pkg) - - def __repr__(self): - return "<%s pkg=%r wrapped=%r @%#8x>" % ( - self.__class__.__name__, self._raw_pkg, self._configurable_name, - id(self)) - - def freeze(self): - o = copy.copy(self) - o.lock() - return o - - def lock(self): - """ - commit any outstanding changes and lock the configuration. - """ - self.commit() - object.__setattr__(self, '_configurable', list(self._configurable)) - - def build(self, **kwds): - if self._buildable: - return self._buildable(self, **kwds) - return None - - return PackageWrapper diff --git a/pkgcore/package/errors.py b/pkgcore/package/errors.py deleted file mode 100644 index 9516871..0000000 --- a/pkgcore/package/errors.py +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright: 2006 Marien Zwart <marienz@gentoo.org> -# License: GPL2 - -#base class -class InvalidPackage(ValueError): - pass - - -class MetadataException(Exception): - - def __init__(self, pkg, attr, error): - Exception.__init__(self, - "Metadata Exception: pkg %s, attr %s\nerror: %s" % - (pkg, attr, error)) - self.pkg, self.attr, self.error = pkg, attr, error - -class InvalidDependency(ValueError): - pass diff --git a/pkgcore/package/metadata.py b/pkgcore/package/metadata.py deleted file mode 100644 index ffc9501..0000000 --- a/pkgcore/package/metadata.py +++ /dev/null @@ -1,126 +0,0 @@ -# Copyright: 2005-2006 Brian Harring <ferringb@gmail.com> -# License: GPL2 - - -""" -package with its metadata accessible (think 'no longer abstract') -""" - -from pkgcore.ebuild.cpv import CPV -from pkgcore.ebuild.atom import atom - -from snakeoil.weakrefs import WeakValCache - -def DeriveMetadataKls(original_kls): - if getattr(original_kls, "_derived_metadata_kls", False): - return original_kls - - class package(original_kls): - _derived_metadata_kls = True - built = False - __slots__ = ("_parent", "data") - try: - __doc__ = "package class with metadata bound to it for attribute " \ - "generation\n\n" + \ - "\n".join(x.lstrip() - for x in original_kls.__doc__.split("\n") - if "@ivar" in x or "@cvar" in x) - __doc__ += "\n@ivar repo: parent repository" - except AttributeError: - # wee, must be in -OO mode. - __doc__ = None - - immutable = True - package_is_real = True - - _get_attr = dict(original_kls._get_attr) - - def __init__(self, parent_repository, *a, **kwds): - """ - wrapper for %s.__init__; see %s.__init__ for allowed args/kwds, - they're passed directly to it - - @param parent_repository: parent repository this package belongs to - @type parent_repository: L{pkgcore.repository.prototype.tree} - instance - """ - original_kls.__init__(self, *a, **kwds) - object.__setattr__(self, "_parent", parent_repository) - - def _get_data(self): - """ - internal hook func to get the packages metadata, consumer - of L{_get_attr} - """ - return self._fetch_metadata() - _get_attr["data"] = _get_data - - @property - def repo(self): - return self._parent._parent_repo - - @property - def slotted_atom(self): - return atom("%s:%s" % (self.key, self.slot)) - - def _fetch_metadata(self): - """ - pull the metadata for this package. - must be overridden in derivative - """ - raise NotImplementedError - - def add_format_triggers(self, op_inst, format_op_inst, engine_inst): - pass - - return package - -package = DeriveMetadataKls(CPV) - -class factory(object): - - """ - package generator - - does weakref caching per repository - - @cvar child_class: callable to generate packages - """ - - child_class = package - - def __init__(self, parent_repo): - self._parent_repo = parent_repo - self._cached_instances = WeakValCache() - - def new_package(self, *args): - """ - generate a new package instance - - """ - inst = self._cached_instances.get(args) - if inst is None: - inst = self._cached_instances[args] = self.child_class(self, *args) - return inst - - def __call__(self, *args, **kwds): - return self.new_package(*args, **kwds) - - def clear(self): - """ - wipe the weakref cache of packages instances - """ - self._cached_instances.clear() - - def _get_metadata(self, *args): - """Pulls metadata from the repo/cache/wherever. - - Must be overriden in derivatives. - """ - raise NotImplementedError - - def _update_metadata(self, *args): - """Updates metadata in the repo/cache/wherever. - - Must be overriden in derivatives.""" - raise NotImplementedError diff --git a/pkgcore/package/mutated.py b/pkgcore/package/mutated.py deleted file mode 100644 index 03da7a7..0000000 --- a/pkgcore/package/mutated.py +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright: 2006 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -""" -package wrapper class to override a packages attributes -""" - -from pkgcore.package.base import wrapper - -class MutatedPkg(wrapper): - __slots__ = ("_overrides",) - - def __init__(self, pkg, overrides): - """ - @param pkg: L{pkgcore.package.metadata.package} to wrap - @param overrides: is an attr -> instance mapping to substitute when - the attr is requested - """ - wrapper.__init__(self, pkg) - object.__setattr__(self, "_overrides", overrides) - - def __getattr__(self, attr): - o = self._overrides.get(attr) - if o is not None: - return o - return getattr(self._raw_pkg, attr) - - def __repr__(self): - return '<%s pkg=%r overrides=%r @%#8x>' % ( - self.__class__.__name__, self._raw_pkg, tuple(self._overrides), - id(self)) - - def __str__(self): - return '%s(%s, overrides=%s)' % \ - (self.__class__.__name__, self._raw_pkg, tuple(self._overrides)) - - @property - def versioned_atom(self): - return self._raw_pkg.versioned_atom - - @property - def unversioned_atom(self): - return self._raw_pkg.unversioned_atom diff --git a/pkgcore/package/virtual.py b/pkgcore/package/virtual.py deleted file mode 100644 index 813abb4..0000000 --- a/pkgcore/package/virtual.py +++ /dev/null @@ -1,49 +0,0 @@ -# Copyright: 2005 Jason Stubbs <jstubbs@gentoo.org> -# License: GPL2 - -""" -virtual package -""" - -from pkgcore.package import metadata -from pkgcore.restrictions.packages import OrRestriction - -class package(metadata.package): - - """ - Virtual package. - - Mainly useful since it's generating so little attrs on the fly. - """ - - package_is_real = False - built = True - - __slots__ = ("__dict__") - - def __init__(self, repo, provider, *a, **kwds): - metadata.package.__init__(self, repo, *a, **kwds) - object.__setattr__(self, 'provider', provider) - object.__setattr__(self, 'data', {}) - - def __getattr__ (self, key): - val = None - if key == "rdepends": - val = self.provider - elif key in ("depends", "post_rdepends", "provides"): - val = OrRestriction(finalize=True) - elif key == "slot": - val = "%s-%s" % (self.provider.category, self.version) - else: - return super(package, self).__getattr__(key) - self.__dict__[key] = val - return val - - def _fetch_metadata(self): - data = self._parent._parent_repo._fetch_metadata(self) - return data - - -class factory(metadata.factory): - child_class = package - diff --git a/pkgcore/pkgsets/__init__.py b/pkgcore/pkgsets/__init__.py deleted file mode 100644 index e0a075b..0000000 --- a/pkgcore/pkgsets/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright: 2006 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -""" -restriction generaters representing sets of packages -""" diff --git a/pkgcore/pkgsets/filelist.py b/pkgcore/pkgsets/filelist.py deleted file mode 100644 index 346d3ba..0000000 --- a/pkgcore/pkgsets/filelist.py +++ /dev/null @@ -1,78 +0,0 @@ -# Copyright: 2006 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -""" -pkgset based around loading a list of atoms from a world file -""" - -import pkgcore.const -from pkgcore.ebuild.atom import atom -from pkgcore.config import ConfigHint - -from snakeoil.demandload import demandload -demandload(globals(), - 'snakeoil.fileutils:AtomicWriteFile', - 'snakeoil.osutils:readlines', - 'pkgcore:os_data', -) - -class FileList(object): - pkgcore_config_type = ConfigHint({'location':'str'}, typename='pkgset') - - def __init__(self, location): - self.path = location - # note that _atoms is generated on the fly. - - def __getattr__(self, attr): - if attr != "_atoms": - raise AttributeError(attr) - s = set() - for x in readlines(self.path): - x = x.strip() - if not x: - continue - s.add(atom(x)) - self._atoms = s - return s - - def __iter__(self): - return iter(self._atoms) - - def __len__(self): - return len(self._atoms) - - def __contains__(self, key): - return key in self._atoms - - def add(self, atom_inst): - self._atoms.add(atom_inst) - - def remove(self, atom_inst): - self._atoms.remove(atom_inst) - - def flush(self): - f = None - # structured this way to force deletion (thus wiping) if something - # fails. - try: - f = AtomicWriteFile(self.path, gid=os_data.portage_gid, perms=0644) - f.write("\n".join(map(str, self._atoms))) - f.close() - finally: - del f - - -class WorldFile(FileList): - pkgcore_config_type = ConfigHint(typename='pkgset') - - def __init__(self, location=pkgcore.const.WORLD_FILE): - FileList.__init__(self, location) - - def add(self, atom_inst): - atom_inst = atom(atom_inst.key) - FileList.add(self, atom_inst) - - def remove(self, atom_inst): - atom_inst = atom(atom_inst.key) - FileList.remove(self, atom_inst) - diff --git a/pkgcore/pkgsets/glsa.py b/pkgcore/pkgsets/glsa.py deleted file mode 100644 index 0d57142..0000000 --- a/pkgcore/pkgsets/glsa.py +++ /dev/null @@ -1,253 +0,0 @@ -# Copyright: 2006 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -""" -Gentoo Linux Security Advisories (GLSA) support -""" - -import os - -from pkgcore.restrictions import packages, restriction, boolean, values -from pkgcore.config import ConfigHint - -from snakeoil.osutils import listdir_files, join as pjoin -from snakeoil.klass import generic_equality -from snakeoil.iterables import caching_iter -from snakeoil.demandload import demandload -demandload(globals(), - 'pkgcore.package:mutated', - 'pkgcore.ebuild:cpv,atom', - 'pkgcore.log:logger', - 'pkgcore.util.repo_utils:get_virtual_repos', - 'snakeoil.xml:etree', -) - - -class KeyedAndRestriction(boolean.AndRestriction): - - type = packages.package_type - - def __init__(self, *a, **kwds): - key = kwds.pop("key", None) - tag = kwds.pop("tag", None) - boolean.AndRestriction.__init__(self, *a, **kwds) - object.__setattr__(self, "key", key) - object.__setattr__(self, "tag", tag) - - def __str__(self): - if self.tag is None: - return boolean.AndRestriction.__str__(self) - return "%s %s" % (self.tag, boolean.AndRestriction.__str__(self)) - - -class GlsaDirSet(object): - - """ - generate a pkgset based on GLSA's distributed via a directory. - - (rsync tree is the usual source.) - """ - - pkgcore_config_type = ConfigHint({'src': 'ref:repo'}, typename='pkgset') - op_translate = {"ge":">=", "gt":">", "lt":"<", "le":"<=", "eq":"="} - - __metaclass__ = generic_equality - __attr_comparison__ = ('paths',) - - def __init__(self, src): - """ - @param src: where to get the glsa from - @type src: must be either full path to glsa dir, or a repo object - to pull it from - """ - - if not isinstance(src, basestring): - src = tuple(sorted(filter(os.path.isdir, - (pjoin(repo.base, 'metadata', 'glsa') for repo in - get_virtual_repos(src, False) if hasattr(repo, 'base')) - ))) - else: - src = [src] - self.paths = src - - def __iter__(self): - for glsa, catpkg, pkgatom, vuln in self.iter_vulnerabilities(): - yield KeyedAndRestriction(pkgatom, vuln, finalize=True, key=catpkg, - tag="GLSA vulnerable:") - - def pkg_grouped_iter(self, sorter=None): - """ - yield GLSA restrictions grouped by package key - - @param sorter: must be either None, or a comparison function - """ - - if sorter is None: - sorter = iter - pkgs = {} - pkgatoms = {} - for glsa, pkg, pkgatom, vuln in self.iter_vulnerabilities(): - pkgatoms[pkg] = pkgatom - pkgs.setdefault(pkg, []).append(vuln) - - for pkgname in sorter(pkgs): - yield KeyedAndRestriction(pkgatoms[pkgname], - packages.OrRestriction(*pkgs[pkgname]), - key=pkgname) - - - def iter_vulnerabilities(self): - """ - generator yielding each GLSA restriction - """ - for path in self.paths: - for fn in listdir_files(path): - #"glsa-1234-12.xml - if not (fn.startswith("glsa-") and fn.endswith(".xml")): - continue - # This verifies the filename is of the correct syntax. - try: - [int(x) for x in fn[5:-4].split("-")] - except ValueError: - continue - root = etree.parse(pjoin(path, fn)) - glsa_node = root.getroot() - if glsa_node.tag != 'glsa': - raise ValueError("glsa without glsa rootnode") - for affected in root.findall('affected'): - for pkg in affected.findall('package'): - try: - pkgname = str(pkg.get('name')).strip() - pkg_vuln_restrict = \ - self.generate_intersects_from_pkg_node( - pkg, tag="glsa(%s)" % fn[5:-4]) - if pkg_vuln_restrict is None: - continue - pkgatom = atom.atom(pkgname) - yield fn[5:-4], pkgname, pkgatom, pkg_vuln_restrict - except (TypeError, ValueError), v: - # thrown from cpv. - logger.warn("invalid glsa- %s, package %s: error %s" - % (fn, pkgname, v)) - del v - - - def generate_intersects_from_pkg_node(self, pkg_node, tag=None): - arch = pkg_node.get("arch") - if arch is not None: - arch = str(arch.strip()).split() - if not arch or "*" in arch: - arch = None - - vuln = list(pkg_node.findall("vulnerable")) - if not vuln: - return None - elif len(vuln) > 1: - vuln_list = [self.generate_restrict_from_range(x) for x in vuln] - vuln = packages.OrRestriction(finalize=True, *vuln_list) - else: - vuln_list = [self.generate_restrict_from_range(vuln[0])] - vuln = vuln_list[0] - if arch is not None: - vuln = packages.AndRestriction(vuln, packages.PackageRestriction( - "keywords", values.ContainmentMatch(all=False, *arch))) - invuln = (pkg_node.findall("unaffected")) - if not invuln: - # wrap it. - return KeyedAndRestriction(vuln, tag=tag, finalize=True) - invuln_list = [self.generate_restrict_from_range(x, negate=True) - for x in invuln] - invuln = [x for x in invuln_list if x not in vuln_list] - if not invuln: - if tag is None: - return KeyedAndRestriction(vuln, tag=tag, finalize=True) - return KeyedAndRestriction(vuln, tag=tag, finalize=True) - return KeyedAndRestriction(vuln, finalize=True, tag=tag, *invuln) - - def generate_restrict_from_range(self, node, negate=False): - op = str(node.get("range").strip()) - base = str(node.text.strip()) - glob = base.endswith("*") - if glob: - base = base[:-1] - base = cpv.CPV("cat/pkg-%s" % base) - restrict = self.op_translate[op.lstrip("r")] - if op.startswith("r"): - if glob: - raise ValueError("glob cannot be used with %s ops" % op) - elif not base.revision: - if '=' not in restrict: - # this is a non-range. - raise ValueError( - "range %s version %s is a guranteed empty set" % - (op, str(node.text.strip()))) - return atom.VersionMatch("~", base.version, negate=negate) - return packages.AndRestriction( - atom.VersionMatch("~", base.version), - atom.VersionMatch(restrict, base.version, rev=base.revision), - finalize=True, negate=negate) - if glob: - return packages.PackageRestriction("fullver", - values.StrGlobMatch(base.fullver)) - return atom.VersionMatch(restrict, base.version, rev=base.revision, - negate=negate) - - -def find_vulnerable_repo_pkgs(glsa_src, repo, grouped=False, arch=None): - """ - generator yielding GLSA restrictions, and vulnerable pkgs from a repo. - - @param glsa_src: GLSA pkgset to pull vulnerabilities from - @param repo: repo to scan for vulnerable packages - @param grouped: if grouped, combine glsa restrictions into one restriction - (thus yielding a pkg only once) - @param arch: arch to scan for, x86 for example - """ - - if grouped: - i = glsa_src.pkg_grouped_iter() - else: - i = iter(glsa_src) - if arch is None: - wrapper = lambda p: p - else: - if isinstance(arch, basestring): - arch = (arch,) - else: - arch = tuple(arch) - wrapper = lambda p: mutated.MutatedPkg(p, {"keywords":arch}) - for restrict in i: - matches = caching_iter(wrapper(x) - for x in repo.itermatch(restrict, - sorter=sorted)) - if matches: - yield restrict, matches - - -class SecurityUpgrades(object): - - """ - pkgset that can be used directly from pkgcore configuration. - - generates set of restrictions of required upgrades. - """ - - pkgcore_config_type = ConfigHint({'ebuild_repo': 'ref:repo', - 'vdb': 'ref:vdb'}, - typename='pkgset') - - __metaclass__ = generic_equality - __attr_comparison__ = ('arch', 'glsa_src', 'vdb') - - def __init__(self, ebuild_repo, vdb, arch): - self.glsa_src = GlsaDirSet(ebuild_repo) - self.vdb = vdb - self.arch = arch - - def __iter__(self): - for glsa, matches in find_vulnerable_repo_pkgs(self.glsa_src, self.vdb, - grouped=True, - arch=self.arch): - yield KeyedAndRestriction(glsa[0], restriction.Negate(glsa[1]), - finalize=True) - diff --git a/pkgcore/pkgsets/installed.py b/pkgcore/pkgsets/installed.py deleted file mode 100644 index 3190ea6..0000000 --- a/pkgcore/pkgsets/installed.py +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright: 2006 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -import operator - -from pkgcore.restrictions import packages, values -from pkgcore.config import ConfigHint - - -class _Base(object): - - """Base for Installed and VersionedInstalled.""" - - def __init__(self, vdb): - self.vdbs = vdb - - def __iter__(self): - restrict = packages.PackageRestriction("package_is_real", - values.EqualityMatch(True)) - for repo in self.vdbs: - for pkg in repo.itermatch(restrict): - yield self.getter(pkg) - - -class Installed(_Base): - - """pkgset holding slotted_atoms of all installed pkgs.""" - - pkgcore_config_type = ConfigHint({'vdb': 'refs:repo'}, typename='pkgset') - getter = operator.attrgetter('slotted_atom') - - -class VersionedInstalled(_Base): - - """pkgset holding versioned_atoms of all installed pkgs.""" - - pkgcore_config_type = ConfigHint({'vdb': 'refs:repo'}, typename='pkgset') - getter = operator.attrgetter('versioned_atom') diff --git a/pkgcore/pkgsets/system.py b/pkgcore/pkgsets/system.py deleted file mode 100644 index 978d164..0000000 --- a/pkgcore/pkgsets/system.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright: 2006 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -""" -system pkgset based off of profile system collapsing -""" - -# yuck. :) -from pkgcore.config import configurable - -@configurable({'profile': 'ref:profile'}, typename='pkgset') -def SystemSet(profile): - return frozenset(profile.system) diff --git a/pkgcore/plugin.py b/pkgcore/plugin.py deleted file mode 100644 index 07d3c93..0000000 --- a/pkgcore/plugin.py +++ /dev/null @@ -1,224 +0,0 @@ -# Copyright: 2006 Marien Zwart <marienz@gentoo.org> -# License: GPL2 - - -"""Plugin system, heavily inspired by twisted's plugin system.""" - -# Implementation note: we have to be pretty careful about error -# handling in here since some core functionality in pkgcore uses this -# code. Since we can function without a cache we will generally be -# noisy but keep working if something is wrong with the cache. -# -# Currently we explode if something is wrong with a plugin package -# dir, but not if something prevents importing a module in it. -# Rationale is the former should be a PYTHONPATH issue while the -# latter an installed plugin issue. May have to change this if it -# causes problems. - -import operator -import os.path - -from pkgcore import plugins -from snakeoil.osutils import join as pjoin -from snakeoil import modules, demandload -demandload.demandload(globals(), 'tempfile', 'errno', 'pkgcore.log:logger') - - -CACHE_HEADER = 'pkgcore plugin cache v2\n' - -# Global plugin cache. Mapping of package to package cache, which is a -# mapping of plugin key to a list of module names. -_cache = {} - - -def initialize_cache(package): - """Determine available plugins in a package. - - Writes cache files if they are stale and writing is possible. - """ - # package plugin cache, see above. - package_cache = {} - seen_modnames = set() - for path in package.__path__: - # Check if the path actually exists first. - try: - modlist = os.listdir(path) - except OSError, e: - if e.errno != errno.ENOENT: - raise - continue - # Directory cache, mapping modulename to - # (mtime, set([keys])) - stored_cache = {} - stored_cache_name = pjoin(path, 'plugincache2') - try: - cachefile = open(stored_cache_name) - except IOError: - # Something is wrong with the cache file. We just handle - # this as a missing/empty cache, which will force a - # rewrite. If whatever it is that is wrong prevents us - # from writing the new cache we log it there. - pass - else: - try: - # Remove this extra nesting once we require python 2.5 - try: - if cachefile.readline() != CACHE_HEADER: - raise ValueError('bogus header') - for line in cachefile: - module, mtime, entries = line[:-1].split(':', 2) - mtime = int(mtime) - result = set() - # Needed because ''.split(':') == [''], not [] - if entries: - for s in entries.split(':'): - name, max_prio = s.split(',') - if max_prio: - max_prio = int(max_prio) - else: - max_prio = None - result.add((name, max_prio)) - stored_cache[module] = (mtime, result) - except ValueError: - # Corrupt cache, treat as empty. - stored_cache = {} - finally: - cachefile.close() - cache_stale = False - # Hunt for modules. - actual_cache = {} - assumed_valid = set() - for modfullname in modlist: - modname, modext = os.path.splitext(modfullname) - if modext != '.py': - continue - if modname == '__init__': - continue - if modname in seen_modnames: - # This module is shadowed by a module earlier in - # sys.path. Skip it, assuming its cache is valid. - assumed_valid.add(modname) - continue - # It is an actual module. Check if its cache entry is valid. - mtime = int(os.path.getmtime(pjoin(path, modfullname))) - if mtime == stored_cache.get(modname, (0, ()))[0]: - # Cache is good, use it. - actual_cache[modname] = stored_cache[modname] - else: - # Cache entry is stale. - logger.debug( - 'stale because of %s: actual %s != stored %s', - modname, mtime, stored_cache.get(modname, (0, ()))[0]) - cache_stale = True - entries = [] - qualname = '.'.join((package.__name__, modname)) - try: - module = modules.load_module(qualname) - except modules.FailedImport: - # This is a serious problem, but if we blow up - # here we cripple pkgcore entirely which may make - # fixing the problem impossible. So be noisy but - # try to continue. - logger.exception('plugin import failed') - else: - values = set() - registry = getattr(module, 'pkgcore_plugins', {}) - for key, plugs in registry.iteritems(): - max_prio = None - for plug in plugs: - priority = getattr(plug, 'priority', None) - if priority is not None \ - and not isinstance(priority, int): - # This happens rather a lot with - # plugins not meant for use with - # get_plugin. Just ignore it. - priority = None - if priority is not None and ( - max_prio is None or priority > max_prio): - max_prio = priority - values.add((key, max_prio)) - actual_cache[modname] = (mtime, values) - # Cache is also stale if it sees entries that are no longer there. - for key in stored_cache: - if key not in actual_cache and key not in assumed_valid: - logger.debug('stale because %s is no longer there', key) - cache_stale = True - break - if cache_stale: - # Write a new cache. - try: - fd, name = tempfile.mkstemp(dir=path) - except OSError, e: - # We cannot write a new cache. We should log this - # since it will have a performance impact. - - # Use error, not exception for this one: the traceback - # is not necessary and too alarming. - logger.error('Cannot write cache for %s: %s. ' - 'Try running pplugincache.', - stored_cache_name, e) - else: - cachefile = os.fdopen(fd, 'w') - cachefile.write(CACHE_HEADER) - try: - for module, (mtime, entries) in actual_cache.iteritems(): - strings = [] - for plugname, max_prio in entries: - if max_prio is None: - strings.append(plugname + ',') - else: - strings.append('%s,%s' % (plugname, max_prio)) - cachefile.write( - '%s:%s:%s\n' % (module, mtime, ':'.join(strings))) - finally: - cachefile.close() - os.chmod(name, 0644) - os.rename(name, stored_cache_name) - # Update the package_cache. - for module, (mtime, entries) in actual_cache.iteritems(): - seen_modnames.add(module) - for key, max_prio in entries: - package_cache.setdefault(key, []).append((module, max_prio)) - return package_cache - - -def get_plugins(key, package=plugins): - """Return all enabled plugins matching "key". - - Plugins with a C{disabled} attribute evaluating to C{True} are skipped. - """ - cache = _cache.get(package) - if cache is None: - cache = _cache[package] = initialize_cache(package) - for modname, max_prio in cache.get(key, ()): - module = modules.load_module('.'.join((package.__name__, modname))) - for obj in module.pkgcore_plugins.get(key, ()): - if not getattr(obj, 'disabled', False): - yield obj - - -def get_plugin(key, package=plugins): - """Get a single plugin matching this key. - - This assumes all plugins for this key have a priority attribute. - If any of them do not the AttributeError is not stopped. - - @return: highest-priority plugin or None if no plugin available. - """ - cache = _cache.get(package) - if cache is None: - cache = _cache[package] = initialize_cache(package) - modlist = cache.get(key, []) - modlist.sort(key=operator.itemgetter(1), reverse=True) - plugs = [] - for i, (modname, max_prio) in enumerate(modlist): - module = modules.load_module('.'.join((package.__name__, modname))) - plugs.extend( - plug for plug in module.pkgcore_plugins.get(key, ()) - if not getattr(plug, 'disabled', False)) - if not plugs: - continue - plugs.sort(key=operator.attrgetter('priority'), reverse=True) - if i + 1 == len(modlist) or plugs[0].priority > modlist[i + 1][1]: - return plugs[0] - return None diff --git a/pkgcore/plugins/.gitignore b/pkgcore/plugins/.gitignore deleted file mode 100644 index 2a3520a..0000000 --- a/pkgcore/plugins/.gitignore +++ /dev/null @@ -1 +0,0 @@ -plugincache2 diff --git a/pkgcore/plugins/__init__.py b/pkgcore/plugins/__init__.py deleted file mode 100644 index 0e8743d..0000000 --- a/pkgcore/plugins/__init__.py +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright: 2006 Marien Zwart <marienz@gentoo.org> -# License: GPL2 - - -"""pkgcore plugins package.""" - -import sys -import os.path - - -# XXX Having this function here is a bit of a wart: it is used by -# other plugin packages (like the pkgcore-check one), but we cannot -# put it in pkgcore.plugin because that imports this package (circular -# import). - -def extend_path(path, name): - """Simpler version of the stdlib's L{pkgutil.extend_path}. - - It does not support ".pkg" files, and it does not require an - __init__.py (this is important: we want only one thing (pkgcore - itself) to install the __init__.py to avoid name clashes). - - It also modifies the "path" list in place (and returns C{None}) - instead of copying it and returning the modified copy. - """ - if not isinstance(path, list): - # This could happen e.g. when this is called from inside a - # frozen package. Return the path unchanged in that case. - return - # Reconstitute as relative path. - pname = os.path.join(*name.split('.')) - - for entry in sys.path: - if not isinstance(entry, basestring) or not os.path.isdir(entry): - continue - subdir = os.path.join(entry, pname) - # XXX This may still add duplicate entries to path on - # case-insensitive filesystems - if subdir not in path: - path.append(subdir) - -extend_path(__path__, __name__) diff --git a/pkgcore/plugins/pkgcore_configurables.py b/pkgcore/plugins/pkgcore_configurables.py deleted file mode 100644 index 27e63e0..0000000 --- a/pkgcore/plugins/pkgcore_configurables.py +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright: 2006 Marien Zwart <marienz@gentoo.org> -# License: GPL2 - -from pkgcore.config import basics -from pkgcore.ebuild import ( - portage_conf, repository as ebuild_repo, profiles, domain, eclass_cache, - overlay_repository, formatter) -from pkgcore.pkgsets import system, filelist, installed, glsa -from pkgcore.vdb import ondisk -from pkgcore.cache import flat_hash, metadata -from pkgcore.fetch import custom -from pkgcore.binpkg import repository as binpkg_repo -from pkgcore.sync import rsync, base - - -pkgcore_plugins = { - 'configurable': [ - basics.section_alias, - basics.parse_config_file, - portage_conf.SecurityUpgradesViaProfile, - portage_conf.config_from_make_conf, - system.SystemSet, - ondisk.tree, - flat_hash.database, - metadata.database, - metadata.paludis_flat_list, - custom.fetcher, - binpkg_repo.tree, - ebuild_repo.UnconfiguredTree, - ebuild_repo.SlavedTree, - profiles.OnDiskProfile, - domain.domain, - eclass_cache.cache, - eclass_cache.StackedCaches, - overlay_repository.OverlayRepo, - formatter.basic_factory, - formatter.pkgcore_factory, - formatter.portage_factory, - formatter.paludis_factory, - formatter.portage_verbose_factory, - filelist.FileList, - filelist.WorldFile, - installed.Installed, - installed.VersionedInstalled, - glsa.GlsaDirSet, - glsa.SecurityUpgrades, - rsync.rsync_syncer, - base.GenericSyncer, - ], - } diff --git a/pkgcore/plugins/pkgcore_ebuild_built.py b/pkgcore/plugins/pkgcore_ebuild_built.py deleted file mode 100644 index 73b42c7..0000000 --- a/pkgcore/plugins/pkgcore_ebuild_built.py +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright: 2007 Marien Zwart <marienz@gentoo.org> -# License: GPL2 - -from pkgcore.ebuild import ebuild_built - -pkgcore_plugins = { - 'format.ebuild_built': [ebuild_built.generate_new_factory], - } diff --git a/pkgcore/plugins/pkgcore_ebuild_src.py b/pkgcore/plugins/pkgcore_ebuild_src.py deleted file mode 100644 index 164e1e7..0000000 --- a/pkgcore/plugins/pkgcore_ebuild_src.py +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright: 2007 Marien Zwart <marienz@gentoo.org> -# License: GPL2 - -from pkgcore.ebuild import ebuild_src - -pkgcore_plugins = { - 'format.ebuild_src': [ebuild_src.generate_new_factory], - } diff --git a/pkgcore/plugins/pkgcore_formatters.py b/pkgcore/plugins/pkgcore_formatters.py deleted file mode 100644 index 7308d36..0000000 --- a/pkgcore/plugins/pkgcore_formatters.py +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright: 2006 Marien Zwart <marienz@gentoo.org> -# License: GPL2 - -from pkgcore.config import basics - -pkgcore_plugins = { - 'global_config': [{ - 'basic-formatter': basics.ConfigSectionFromStringDict({ - 'class': 'pkgcore.ebuild.formatter.basic_factory', - }), - 'pkgcore-formatter': basics.ConfigSectionFromStringDict({ - 'class': 'pkgcore.ebuild.formatter.pkgcore_factory', - }), - 'portage-formatter': basics.ConfigSectionFromStringDict({ - 'class': 'pkgcore.ebuild.formatter.portage_factory', - 'default': 'True', - }), - 'paludis-formatter': basics.ConfigSectionFromStringDict({ - 'class': 'pkgcore.ebuild.formatter.paludis_factory', - }), - 'portage-verbose-formatter': basics.ConfigSectionFromStringDict({ - 'class': - 'pkgcore.ebuild.formatter.portage_verbose_factory', - }), - }], - } diff --git a/pkgcore/plugins/pkgcore_fsops_default.py b/pkgcore/plugins/pkgcore_fsops_default.py deleted file mode 100644 index 717d740..0000000 --- a/pkgcore/plugins/pkgcore_fsops_default.py +++ /dev/null @@ -1,12 +0,0 @@ -# Copyright: 2006 Marien Zwart <marienz@gentoo.org> -# License: GPL2 - -from pkgcore.fs import ops - -pkgcore_plugins = { - 'fs_ops.copyfile': [ops.default_copyfile], - 'fs_ops.ensure_perms': [ops.default_ensure_perms], - 'fs_ops.mkdir': [ops.default_mkdir], - 'fs_ops.merge_contents': [ops.merge_contents], - 'fs_ops.unmerge_contents': [ops.unmerge_contents], - } diff --git a/pkgcore/plugins/pkgcore_syncers.py b/pkgcore/plugins/pkgcore_syncers.py deleted file mode 100644 index 10a4313..0000000 --- a/pkgcore/plugins/pkgcore_syncers.py +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright: 2006 Marien Zwart <marienz@gentoo.org> -# License: GPL2 - -from pkgcore.sync import bzr, cvs, darcs, git, hg, svn - - -pkgcore_plugins = { - 'syncer': [ - bzr.bzr_syncer, - cvs.cvs_syncer, - darcs.darcs_syncer, - git.git_syncer, - hg.hg_syncer, - svn.svn_syncer, - ], - } diff --git a/pkgcore/plugins/pkgcore_triggers.py b/pkgcore/plugins/pkgcore_triggers.py deleted file mode 100644 index 7cca4f2..0000000 --- a/pkgcore/plugins/pkgcore_triggers.py +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright: 2006 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -from pkgcore.merge import triggers - -pkgcore_plugins = { - 'triggers':[ - triggers.ldconfig, - triggers.merge, - triggers.unmerge, - triggers.fix_uid_perms, - triggers.fix_gid_perms, - triggers.fix_set_bits, - triggers.detect_world_writable, - triggers.InfoRegen, - triggers.CommonDirectoryModes, - ], - } diff --git a/pkgcore/repository/__init__.py b/pkgcore/repository/__init__.py deleted file mode 100644 index 4982be7..0000000 --- a/pkgcore/repository/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright: 2005 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -""" -repository subsystem -""" diff --git a/pkgcore/repository/configured.py b/pkgcore/repository/configured.py deleted file mode 100644 index 1f2ba0e..0000000 --- a/pkgcore/repository/configured.py +++ /dev/null @@ -1,62 +0,0 @@ -# Copyright: 2006 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -""" -wrap a repository, binding configuration to pkgs returned from the repository -""" - -from pkgcore.repository import prototype -from pkgcore.package.conditionals import make_wrapper -from snakeoil.currying import partial -from snakeoil.klass import GetAttrProxy - - -class tree(prototype.tree): - configured = True - - def __init__(self, raw_repo, wrapped_attrs): - - """ - @param raw_repo: repo to wrap - @type raw_repo: L{pkgcore.repository.prototype.tree} - @param wrapped_attrs: sequence of attrs to wrap for each pkg - """ - - # yes, we're intentionally not using tree's init. - # not perfect I know. - self.raw_repo = raw_repo - self.wrapped_attrs = wrapped_attrs - self.attr_filters = frozenset(wrapped_attrs.keys() + - [self.configurable]) - - self._klass = make_wrapper(self.configurable, self.wrapped_attrs) - - def _get_pkg_kwds(self, pkg): - raise NotImplementedError() - - def package_class(self, pkg, *a): - return self._klass(pkg, **self._get_pkg_kwds(pkg)) - - __getattr__ = GetAttrProxy("raw_repo") - - def itermatch(self, restrict, **kwds): - kwds.setdefault("force", True) - o = kwds.get("pkg_klass_override") - if o is not None: - kwds["pkg_klass_override"] = partial(self.package_class, o) - else: - kwds["pkg_klass_override"] = self.package_class - return self.raw_repo.itermatch(restrict, **kwds) - - itermatch.__doc__ = prototype.tree.itermatch.__doc__.replace( - "@param", "@keyword").replace("@keyword restrict:", "@param restrict:") - - def __getitem__(self, key): - return self.package_class(self.raw_repo[key]) - - def __repr__(self): - return '<%s.%s raw_repo=%r wrapped=%r @%#8x>' % ( - self.__class__.__module__, self.__class__.__name__, - getattr(self, 'raw_repo', 'unset'), - getattr(self, 'wrapped_attrs', {}).keys(), - id(self)) diff --git a/pkgcore/repository/errors.py b/pkgcore/repository/errors.py deleted file mode 100644 index 0cf7b33..0000000 --- a/pkgcore/repository/errors.py +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright: 2005 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -""" -exceptions thrown by repository classes. - -Need to extend the usage a bit further still. -""" - -class TreeCorruption(Exception): - def __init__(self, err): - Exception.__init__(self, "unexpected tree corruption: %s" % (err,)) - self.err = err - -class InitializationError(TreeCorruption): - def __str__(self): - return "initialization failed: %s" % str(self.err) diff --git a/pkgcore/repository/misc.py b/pkgcore/repository/misc.py deleted file mode 100644 index 5be10b7..0000000 --- a/pkgcore/repository/misc.py +++ /dev/null @@ -1,90 +0,0 @@ -# Copyright: 2006-2007 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -from pkgcore.restrictions import packages -from pkgcore.package.mutated import MutatedPkg -from snakeoil.iterables import caching_iter -from snakeoil.klass import GetAttrProxy - -__all__ = ("nodeps_repo", "caching_repo") - -class nodeps_repo(object): - - """ - repository wrapper that returns wrapped pkgs via - L{MutatedPkg} that have their depends/rdepends/post_rdepends wiped - """ - - default_depends = packages.AndRestriction(finalize=True) - default_rdepends = packages.AndRestriction(finalize=True) - default_post_rdepends = packages.AndRestriction(finalize=True) - - def __init__(self, repo): - """ - @param repo: repository to wrap - """ - self.raw_repo = repo - - def itermatch(self, *a, **kwds): - return (MutatedPkg(x, - overrides={"depends":self.default_depends, - "rdepends":self.default_rdepends, - "post_rdepends":self.default_post_rdepends}) - for x in self.raw_repo.itermatch(*a, **kwds)) - - def match(self, *a, **kwds): - return list(self.itermatch(*a, **kwds)) - - __getattr__ = GetAttrProxy("raw_repo") - - def __iter__(self): - return self.itermatch(packages.AlwaysTrue) - - -class caching_repo(object): - - """ - repository wrapper that overrides match, returning - L{caching_iter} instances; itermatch is slaved to match, - in other words iterating over the caching_iter. - - Main use for this is to cache results from query lookups; - if matches restrict arg is in the cache, the caller gets a shared - caching_iter sequence, which may already be fully loaded with pkg - instances. - - This can boost random lookup time pretty nicely, while helping to - hold instance in memory to avoid redoing work. - - Cost of this of course is that involved objects are forced to stay - in memory till the cache is cleared. General use, not usually what - you want- if you're making a lot of random queries that are duplicates - (resolver does this for example), caching helps. - """ - - def __init__(self, db, strategy): - """ - @param db: an instance supporting the repository protocol to cache - queries from. - @param strategy: forced sorting strategy for results. If you don't - need sorting, pass in iter. - """ - self.__db__ = db - self.__strategy__ = strategy - self.__cache__ = {} - - def match(self, restrict): - v = self.__cache__.get(restrict) - if v is None: - v = self.__cache__[restrict] = \ - caching_iter(self.__db__.itermatch(restrict, - sorter=self.__strategy__)) - return v - - def itermatch(self, restrict): - return iter(self.match(restrict)) - - __getattr__ = GetAttrProxy("__db__") - - def clear(self): - self.__cache__.clear() diff --git a/pkgcore/repository/multiplex.py b/pkgcore/repository/multiplex.py deleted file mode 100644 index b054d87..0000000 --- a/pkgcore/repository/multiplex.py +++ /dev/null @@ -1,120 +0,0 @@ -# Copyright: 2005-2006 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -""" -repository that combines multiple repositories together -""" - -from operator import itemgetter -from pkgcore.repository import prototype, errors -from snakeoil.currying import partial -from snakeoil.iterables import iter_sort - -class tree(prototype.tree): - - """repository combining multiple repositories into one""" - - zero_index_grabber = itemgetter(0) - - def __init__(self, *trees): - """ - @param trees: L{pkgcore.repository.prototype.tree} instances - to combines into one - """ - super(tree, self).__init__() - for x in trees: - if not isinstance(x, prototype.tree): - raise errors.InitializationError( - "%s is not a repository tree derivative" % (x,)) - self.trees = trees - - def _get_categories(self, *optional_category): - d = set() - failures = 0 - if optional_category: - optional_category = optional_category[0] - for x in self.trees: - try: - d.update(x.categories[optional_category]) - except KeyError: - failures += 1 - else: - for x in self.trees: - try: - map(d.add, x.categories) - except (errors.TreeCorruption, KeyError): - failures += 1 - if failures == len(self.trees): - if optional_category: - raise KeyError("category base '%s' not found" % - str(optional_category)) - raise KeyError("failed getting categories") - return tuple(d) - - def _get_packages(self, category): - d = set() - failures = 0 - for x in self.trees: - try: - d.update(x.packages[category]) - except (errors.TreeCorruption, KeyError): - failures += 1 - if failures == len(self.trees): - raise KeyError("category '%s' not found" % category) - return tuple(d) - - def _get_versions(self, package): - d = set() - failures = 0 - for x in self.trees: - try: - d.update(x.versions[package]) - except (errors.TreeCorruption, KeyError): - failures += 1 - - if failures == len(self.trees): - raise KeyError("category '%s' not found" % package) - return tuple(d) - - def itermatch(self, restrict, **kwds): - sorter = kwds.get("sorter", iter) - if sorter is iter: - return (match for repo in self.trees - for match in repo.itermatch(restrict, **kwds)) - # ugly, and a bit slow, but works. - def f(x, y): - l = sorter([x, y]) - if l[0] == y: - return 1 - return -1 - f = partial(sorted, cmp=f) - return iter_sort(f, - *[repo.itermatch(restrict, **kwds) for repo in self.trees]) - - itermatch.__doc__ = prototype.tree.itermatch.__doc__.replace( - "@param", "@keyword").replace("@keyword restrict:", "@param restrict:") - - def __iter__(self): - return (pkg for repo in self.trees for pkg in repo) - - def __len__(self): - return sum(len(repo) for repo in self.trees) - - def __getitem__(self, key): - for t in self.trees: - try: - p = t[key] - return p - except KeyError: - pass - # made it here, no match. - raise KeyError("package %s not found" % key) - - def __repr__(self): - return '<%s.%s trees=%r @%#8x>' % ( - self.__class__.__module__, self.__class__.__name__, - getattr(self, 'trees', 'unset'), - id(self)) - - def _visibility_limiters(self): - return [x for r in self.trees for x in r.default_visibility_limiters] diff --git a/pkgcore/repository/prototype.py b/pkgcore/repository/prototype.py deleted file mode 100644 index 5676b49..0000000 --- a/pkgcore/repository/prototype.py +++ /dev/null @@ -1,521 +0,0 @@ -# Copyright: 2005-2007 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -""" -base repository template -""" - -from pkgcore.ebuild.atom import atom -from pkgcore.restrictions import values, boolean, restriction, packages -from pkgcore.restrictions.util import collect_package_restrictions - -from snakeoil.mappings import LazyValDict, DictMixin -from snakeoil.lists import iflatten_instance -from snakeoil.compatibility import any - -class IterValLazyDict(LazyValDict): - - def __str__(self): - return str(list(self)) - - def force_regen(self, key): - if key in self._vals: - del self._vals[key] - else: - self._keys = tuple(x for x in self._keys if x != key) - - -class CategoryIterValLazyDict(IterValLazyDict): - - def force_add(self, key): - if key not in self: - s = set(self._keys) - s.add(key) - self._keys = tuple(s) - - def force_remove(self, key): - if key in self: - self._keys = tuple(x for x in self._keys if x != key) - - __iter__ = IterValLazyDict.iterkeys - - def __contains__(self, key): - if self._keys_func is not None: - return key in self.keys() - return key in self._keys - - -class PackageMapping(DictMixin): - - def __init__(self, parent_mapping, pull_vals): - self._cache = {} - self._parent = parent_mapping - self._pull_vals = pull_vals - - def __getitem__(self, key): - o = self._cache.get(key) - if o is not None: - return o - if key not in self._parent: - raise KeyError(key) - self._cache[key] = vals = self._pull_vals(key) - return vals - - def iterkeys(self): - return self._parent.iterkeys() - - def __contains__(self, key): - return key in self._cache or key in self._parent - - def force_regen(self, cat): - try: - del self._cache[cat] - except KeyError: - pass - - -class VersionMapping(DictMixin): - - def __init__(self, parent_mapping, pull_vals): - self._cache = {} - self._parent = parent_mapping - self._pull_vals = pull_vals - self._known_keys = {} - self._finalized = False - - def __getitem__(self, key): - o = self._cache.get(key) - if o is not None: - return o - cat, pkg = key - known_pkgs = self._known_keys.get(cat) - if known_pkgs is None: - if self._finalized: - raise KeyError(key) - self._known_keys[cat] = known_pkgs = set(self._parent[cat]) - if pkg not in known_pkgs: - raise KeyError(key) - - val = self._pull_vals(key) - self._cache[key] = val - known_pkgs.remove(pkg) - return val - - def iterkeys(self): - for key in self._cache: - yield key - - if not self._finalized: - for cat, pkgs in self._parent.iteritems(): - if cat in self._known_keys: - continue - s = set() - for pkg in pkgs: - if (cat, pkg) in self._cache: - continue - s.add(pkg) - self._known_keys[cat] = s - self._finalized = True - - for cat, pkgs in self._known_keys.iteritems(): - for pkg in list(pkgs): - yield cat, pkg - - def force_regen(self, key, val): - if val: - self._cache[key] = val - else: - self._cache.pop(key, None) - self._known_keys.pop(key[0], None) - - -class tree(object): - """ - repository template - - @ivar raw_repo: if wrapping a repo, set raw_repo per instance to it - @ivar livefs: boolean, set it to True if it's a repository representing - a livefs - @ivar package_class: callable to generate a package instance, must override - @ivar configured: if a repo is unusable for merging/unmerging - without being configured, set it to False - @ivar configure: if the repository isn't configured, must be a callable - yielding a configured form of the repository - """ - - raw_repo = None - livefs = False - package_class = None - configured = True - configure = None - syncable = False - - def __init__(self, frozen=True): - """ - @param frozen: controls whether the repository is mutable or immutable - """ - - self.categories = CategoryIterValLazyDict( - self._get_categories, self._get_categories) - self.packages = PackageMapping(self.categories, - self._get_packages) - self.versions = VersionMapping(self.packages, self._get_versions) - - self.frozen = frozen - self.lock = None - - def _get_categories(self, *args): - """this must return a list, or sequence""" - raise NotImplementedError(self, "_get_categories") - - def _get_packages(self, category): - """this must return a list, or sequence""" - raise NotImplementedError(self, "_get_packages") - - def _get_versions(self, package): - """this must return a list, or sequence""" - raise NotImplementedError(self, "_get_versions") - - def __getitem__(self, cpv): - cpv_inst = self.package_class(*cpv) - if cpv_inst.fullver not in self.versions[(cpv_inst.category, cpv_inst.package)]: - del cpv_inst - raise KeyError(cpv) - return cpv_inst - - def __setitem__(self, *vals): - raise AttributeError - - def __delitem__(self, cpv): - raise AttributeError - - def __iter__(self): - return self.itermatch(packages.AlwaysTrue) - - def __len__(self): - return sum(len(v) for v in self.versions.itervalues()) - - def match(self, atom, **kwds): - return list(self.itermatch(atom, **kwds)) - - def itermatch(self, restrict, restrict_solutions=None, sorter=None, - pkg_klass_override=None, force=None, yield_none=False): - - """ - generator that yields packages match a restriction. - - @type restrict : L{pkgcore.restrictions.packages.PackageRestriction} - instance - @param restrict: restriction to search via - @param restrict_solutions: cnf collapsed list of the restrict. - Don't play with it unless you know what you're doing - @param sorter: callable to do sorting during searching- - if sorting the results, use this instead of sorting externally. - @param yield_none: if True then itermatch will yield None for every - non-matching package. This is meant for use in combination with - C{twisted.task.cooperate} or other async uses where itermatch - should not wait many (wallclock) seconds between yielding - packages. If you override this method you should yield - None in long-running loops, strictly calling it for every package - is not necessary. - """ - - if not isinstance(restrict, restriction.base): - raise TypeError("restrict must be a " - "pkgcore.restriction.restrictions.base instance: " - "got %r" % (restrict,)) - - if sorter is None: - sorter = iter - - if isinstance(restrict, atom): - candidates = [(restrict.category, restrict.package)] - else: - candidates = self._identify_candidates(restrict, sorter) - - if force is None: - match = restrict.match - elif force: - match = restrict.force_True - else: - match = restrict.force_False - return self._internal_match( - candidates, match, sorter, pkg_klass_override, - yield_none=yield_none) - - def _internal_gen_candidates(self, candidates, sorter): - pkls = self.package_class - for cp in candidates: - for pkg in sorter(pkls(cp[0], cp[1], ver) - for ver in self.versions.get(cp, ())): - yield pkg - - def _internal_match(self, candidates, match_func, sorter, - pkg_klass_override, yield_none=False): - for pkg in self._internal_gen_candidates(candidates, sorter): - if pkg_klass_override is not None: - pkg = pkg_klass_override(pkg) - - if match_func(pkg): - yield pkg - elif yield_none: - yield None - - def _identify_candidates(self, restrict, sorter): - # full expansion - - if not isinstance(restrict, boolean.base) or isinstance(restrict, atom): - return self._fast_identify_candidates(restrict, sorter) - dsolutions = [ - ([c.restriction - for c in collect_package_restrictions(x, ["category"])], - [p.restriction - for p in collect_package_restrictions(x, ["package"])]) - for x in restrict.iter_dnf_solutions(True)] - - for x in dsolutions: - if not x[0] and not x[1]: - # great... one doesn't rely on cat/pkg. - if iter is sorter: - return self.versions - return ( - (c,p) - for c in sorter(self.categories) - for p in sorter(self.packages.get(c, ()))) - # simple cases first. - # if one specifies categories, and one doesn't - cat_specified = bool(dsolutions[0][0]) - pkg_specified = bool(dsolutions[0][1]) - pgetter = self.packages.get - if any(True for x in dsolutions[1:] if bool(x[0]) != cat_specified): - if any(True for x in dsolutions[1:] if bool(x[1]) != pkg_specified): - # merde. so we've got a mix- some specify cats, some - # don't, some specify pkgs, some don't. - # this may be optimizable - return self.versions - # ok. so... one doesn't specify a category, but they all - # specify packages (or don't) - pr = values.OrRestriction(*tuple(iflatten_instance( - (x[1] for x in dsolutions if x[1]), values.base))) - return ((c, p) - for c in sorter(self.categories) - for p in sorter(pgetter(c, [])) if pr.match(p)) - - elif any(True for x in dsolutions[1:] if bool(x[1]) != pkg_specified): - # one (or more) don't specify pkgs, but they all specify cats. - cr = values.OrRestriction(*tuple(iflatten_instance( - (x[0] for x in dsolutions), values.base))) - cats_iter = (c for c in sorter(self.categories) if cr.match(c)) - return ((c, p) - for c in cats_iter for p in sorter(pgetter(c, []))) - - return self._fast_identify_candidates(restrict, sorter) - - def _fast_identify_candidates(self, restrict, sorter): - pkg_restrict = set() - cat_restrict = set() - cat_exact = set() - pkg_exact = set() - - for x in collect_package_restrictions(restrict, - ["category", "package"]): - if x.attr == "category": - cat_restrict.add(x.restriction) - elif x.attr == "package": - pkg_restrict.add(x.restriction) - - for e, s in ((pkg_exact, pkg_restrict), (cat_exact, cat_restrict)): - l = [x for x in s - if isinstance(x, values.StrExactMatch) and not x.negate] - s.difference_update(l) - e.update(x.exact for x in l) - del l - - if cat_exact: - if not cat_restrict and len(cat_exact) == 1: - # Cannot use pop here, cat_exact is reused below. - c = iter(cat_exact).next() - if not pkg_restrict and len(pkg_exact) == 1: - cp = (c, pkg_exact.pop()) - if cp in self.versions: - return [cp] - return [] - cats_iter = [c] - else: - cat_restrict.add(values.ContainmentMatch(*cat_exact)) - cats_iter = sorter(self._cat_filter(cat_restrict)) - elif cat_restrict: - cats_iter = self._cat_filter(cat_restrict) - else: - cats_iter = sorter(self.categories) - - if pkg_exact: - if not pkg_restrict: - if sorter is iter: - pkg_exact = tuple(pkg_exact) - else: - pkg_exact = sorter(pkg_exact) - return ( - (c,p) - for c in cats_iter for p in pkg_exact) - else: - pkg_restrict.add(values.ContainmentMatch(*pkg_exact)) - - if pkg_restrict: - return self._package_filter(cats_iter, pkg_restrict) - elif not cat_restrict: - if sorter is iter and not cat_exact: - return self.versions - else: - return ((c, p) for c in - cats_iter for p in sorter(self.packages.get(c, ()))) - return ((c, p) - for c in cats_iter for p in sorter(self.packages.get(c, ()))) - - def _cat_filter(self, cat_restricts): - cats = [x.match for x in cat_restricts] - for x in self.categories: - for match in cats: - if match(x): - yield x - break - - def _package_filter(self, cats_iter, pkg_restricts): - restricts = [x.match for x in pkg_restricts] - pkgs_dict = self.packages - for cat in cats_iter: - for pkg in pkgs_dict.get(cat, ()): - for match in restricts: - if match(pkg): - yield (cat, pkg) - break - - def notify_remove_package(self, pkg): - """ - internal function, - - notify the repository that a pkg it provides is being removed - """ - ver_key = (pkg.category, pkg.package) - l = [x for x in self.versions[ver_key] if x != pkg.fullver] - if not l: - # dead package - wipe = list(self.packages[pkg.category]) == [pkg.package] - self.packages.force_regen(pkg.category) - if wipe: - self.categories.force_regen(pkg.category) - self.versions.force_regen(ver_key, tuple(l)) - - def notify_add_package(self, pkg): - """ - internal function, - - notify the repository that a pkg is being addeded to it - """ - ver_key = (pkg.category, pkg.package) - s = set(self.versions.get(ver_key, ())) - s.add(pkg.fullver) - if pkg.category not in self.categories: - self.categories.force_add(pkg.category) - self.packages.force_regen(pkg.category) - self.versions.force_regen(ver_key, tuple(s)) - - def install(self, pkg, *a, **kw): - """ - internal function, install a pkg to the repository - - @param pkg: L{pkgcore.package.metadata.package} instance to install - @param a: passed to _install - @param kw: passed to _install - @raise AttributeError: if the repository is frozen (immutable) - @return: L{pkgcore.interfaces.repo.nonlivefs_install} or - L{pkgcore.interfaces.repo.livefs_install} instance - """ - if not kw.pop('force', False) and self.frozen: - raise AttributeError("repo is frozen") - return self._install(pkg, *a, **kw) - - def _install(self, pkg, *a, **kw): - """ - internal install function- must be overrided in derivatives - - @param pkg: L{pkgcore.package.metadata.package} instance to install - @param a: passed to _install - @param kw: passed to _install - @return: L{pkgcore.interfaces.repo.nonlivefs_install} or - L{pkgcore.interfaces.repo.livefs_install} instance - """ - raise NotImplementedError(self, "_install") - - def uninstall(self, pkg, *a, **kw): - """ - internal function, uninstall a pkg from the repository - - @param pkg: L{pkgcore.package.metadata.package} instance to install - @param a: passed to _install - @param kw: passed to _install - @raise AttributeError: if the repository is frozen (immutable) - @return: L{pkgcore.interfaces.repo.nonlivefs_uninstall} or - L{pkgcore.interfaces.repo.livefs_uninstall} instance - """ - if self.frozen and not kw.pop("force", False): - raise AttributeError("repo is frozen") - return self._uninstall(pkg, *a, **kw) - - def _uninstall(self, pkg, *a, **kw): - """ - internal uninstall function- must be overrided in derivatives - - @param pkg: L{pkgcore.package.metadata.package} instance to install - @param a: passed to _install - @param kw: passed to _install - @return: L{pkgcore.interfaces.repo.nonlivefs_uninstall} or - L{pkgcore.interfaces.repo.livefs_uninstall} instance - """ - raise NotImplementedError(self, "_uninstall") - - def replace(self, orig, new, *a, **kw): - """ - internal function, replace a pkg in the repository with another - - @param orig: L{pkgcore.package.metadata.package} instance to install, - must be from this repository instance - @param new: L{pkgcore.package.metadata.package} instance to install - @param a: passed to _install - @param kw: passed to _install - @raise AttributeError: if the repository is frozen (immutable) - @return: L{pkgcore.interfaces.repo.nonlivefs_replace} or - L{pkgcore.interfaces.repo.livefs_replace} instance - """ - if self.frozen and not kw.pop("force", False): - raise AttributeError("repo is frozen") - return self._replace(orig, new, *a, **kw) - - def _replace(self, orig, new, *a, **kw): - """ - internal replace function- must be overrided in derivatives - - @param orig: L{pkgcore.package.metadata.package} instance to install, - must be from this repository instance - @param new: L{pkgcore.package.metadata.package} instance to install - @param a: passed to _install - @param kw: passed to _install - @return: L{pkgcore.interfaces.repo.nonlivefs_replace} or - L{pkgcore.interfaces.repo.livefs_replace} instance - """ - raise NotImplementedError(self, "_replace") - - def __nonzero__(self): - try: - iter(self.versions).next() - return True - except StopIteration: - return False - - @property - def default_visibility_limiters(self): - # designed this way to allow for easy override - return self._visibility_limiters() - - def _visibility_limiters(self): - return [] diff --git a/pkgcore/repository/syncable.py b/pkgcore/repository/syncable.py deleted file mode 100644 index 3fb532f..0000000 --- a/pkgcore/repository/syncable.py +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright: 2006 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -from pkgcore.sync import base - -class tree_mixin(object): - - def __init__(self, sync=None): - self._sync = sync - - def sync(self, status_obj=None, force=False): - # often enough, the syncer is a lazy_ref - syncer = self._sync - if not isinstance(syncer, base.syncer): - syncer = syncer.instantiate() - return syncer.sync(force=force) - - @property - def syncable(self): - return self._sync is not None diff --git a/pkgcore/repository/util.py b/pkgcore/repository/util.py deleted file mode 100644 index 4a5cb31..0000000 --- a/pkgcore/repository/util.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright: 2006 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -from pkgcore.repository.prototype import tree -from pkgcore.ebuild.cpv import CPV - -class SimpleTree(tree): - - def __init__(self, cpv_dict, pkg_klass=None): - self.cpv_dict = cpv_dict - if pkg_klass is None: - pkg_klass = CPV - self.package_class = pkg_klass - tree.__init__(self) - - def _get_categories(self, *arg): - if arg: - return () - return tuple(self.cpv_dict.iterkeys()) - - def _get_packages(self, category): - return tuple(self.cpv_dict[category].iterkeys()) - - def _get_versions(self, cp_key): - return tuple(self.cpv_dict[cp_key[0]][cp_key[1]]) - - def notify_remove_package(self, pkg): - vers = self.cpv_dict[pkg.category][pkg.package] - vers = [x for x in vers if x != pkg.fullver] - if vers: - self.cpv_dict[pkg.category][pkg.package] = vers - else: - del self.cpv_dict[pkg.category][pkg.package] - if not self.cpv_dict[pkg.category]: - del self.cpv_dict[pkg.category] - tree.notify_remove_package(self, pkg) - - def notify_add_package(self, pkg): - self.cpv_dict.setdefault(pkg.category, - {}).setdefault(pkg.package, []).append(pkg.fullver) - tree.notify_add_package(self, pkg) diff --git a/pkgcore/repository/virtual.py b/pkgcore/repository/virtual.py deleted file mode 100644 index f021068..0000000 --- a/pkgcore/repository/virtual.py +++ /dev/null @@ -1,51 +0,0 @@ -# Copyright: 2005-2006 Brian harring <ferringb@gmail.com> -# License: GPL2 - -""" -virtual repository, pkgs generated via callable -""" - -from pkgcore.repository import prototype -from pkgcore.package import virtual -from snakeoil.currying import partial - - -class tree(prototype.tree): - - factory_kls = staticmethod(virtual.factory) - - def __init__(self, livefs=False): - """ - @param grab_virtuals_func: callable to get a package -> versions mapping - @param livefs: is this a livefs repository? - """ - prototype.tree.__init__(self) - self.livefs = livefs - vf = self.factory_kls(self) - self.package_class = vf.new_package - - def _expand_vers(self, cp, ver): - raise NotImplementedError(self, "_expand_vers") - - def _internal_gen_candidates(self, candidates, sorter): - pkls = self.package_class - for cp in candidates: - for pkg in sorter(pkls(provider, cp[0], cp[1], ver) - for ver in self.versions.get(cp, ()) - for provider in self._expand_vers(cp, ver)): - yield pkg - - def _get_categories(self, *optional_category): - # return if optional_category is passed... cause it's not yet supported - if optional_category: - return () - return ("virtual",) - - def _load_data(self): - raise NotImplementedError(self, "_load_data") - - def _get_packages(self, category): - if category != "virtual": - raise KeyError("no %s category for this repository" % category) - self._load_data() - return self.packages[category] diff --git a/pkgcore/repository/visibility.py b/pkgcore/repository/visibility.py deleted file mode 100644 index ddb1fae..0000000 --- a/pkgcore/repository/visibility.py +++ /dev/null @@ -1,64 +0,0 @@ -# Copyright: 2005-2006 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -""" -filtering repository -""" - -# icky. -# ~harring -from pkgcore.repository import prototype, errors -from pkgcore.restrictions.restriction import base -from snakeoil.klass import GetAttrProxy - -class filterTree(prototype.tree): - - """Filter existing repository based upon passed in restrictions.""" - - def __init__(self, repo, restriction, sentinel_val=False): - self.raw_repo = repo - self.sentinel_val = sentinel_val - if not isinstance(self.raw_repo, prototype.tree): - raise errors.InitializationError( - "%s is not a repository tree derivative" % (self.raw_repo,)) - if not isinstance(restriction, base): - raise errors.InitializationError( - "%s is not a restriction" % (restriction,)) - self.restriction = restriction - self.raw_repo = repo - - def itermatch(self, restrict, **kwds): - # note that this lets the repo do the initial filtering. - # better design would to analyze the restrictions, and inspect - # the repo, determine what can be done without cost - # (determined by repo's attributes) versus what does cost - # (metadata pull for example). - for cpv in self.raw_repo.itermatch(restrict, **kwds): - if self.restriction.match(cpv) == self.sentinel_val: - yield cpv - - - itermatch.__doc__ = prototype.tree.itermatch.__doc__.replace( - "@param", "@keyword").replace("@keyword restrict:", "@param restrict:") - - def __len__(self): - count = 0 - for i in self: - count += 1 - return count - - __getattr__ = GetAttrProxy("raw_repo") - - def __getitem__(self, key): - v = self.raw_repo[key] - if self.restriction.match(v) != self.sentinel_val: - raise KeyError(key) - return v - - def __repr__(self): - return '<%s raw_repo=%r restriction=%r sentinel=%r @%#8x>' % ( - self.__class__.__name__, - getattr(self, 'raw_repo', 'unset'), - getattr(self, 'restriction', 'unset'), - getattr(self, 'sentinel_val', 'unset'), - id(self)) diff --git a/pkgcore/repository/wrapper.py b/pkgcore/repository/wrapper.py deleted file mode 100644 index 8d4ce90..0000000 --- a/pkgcore/repository/wrapper.py +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright: 2006 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -""" -simple repository wrapping to override the package instances returned -""" - -# icky. -# ~harring -from pkgcore.repository import prototype, errors -from snakeoil.klass import GetAttrProxy -from itertools import imap - -class tree(prototype.tree): - - """wrap an existing repository yielding wrapped packages.""" - - def __init__(self, repo, package_class): - """ - @param repo: L{pkgcore.repository.prototype.tree} instance to wrap - @param package_class: callable to yield the package instance - """ - self.raw_repo = repo - if not isinstance(self.raw_repo, prototype.tree): - raise errors.InitializationError( - "%s is not a repository tree derivative" % (self.raw_repo,)) - self.package_class = package_class - self.raw_repo = repo - - def itermatch(self, *args, **kwargs): - return imap(self.package_class, self.raw_repo.itermatch(*args, **kwargs)) - - __getattr__ = GetAttrProxy("raw_repo") - - def __len__(self): - return len(self.raw_repo) diff --git a/pkgcore/resolver/__init__.py b/pkgcore/resolver/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/pkgcore/resolver/__init__.py +++ /dev/null diff --git a/pkgcore/resolver/choice_point.py b/pkgcore/resolver/choice_point.py deleted file mode 100644 index 68cbf0b..0000000 --- a/pkgcore/resolver/choice_point.py +++ /dev/null @@ -1,155 +0,0 @@ -# Copyright: 2006 Brian Harring <ferringb@gmail.com> -# License: GPL2 - - -from snakeoil.lists import iter_stable_unique - -class choice_point(object): - - __slots__ = ( - "__weakref__", "atom", "matches", "matches_cur", "solution_filters", - "_prdeps", "_rdeps", "_deps", "_provides") - - def __init__(self, a, matches): - self.atom = a - self.matches = iter(matches) - self.matches_cur = None - self.solution_filters = set() - # match solutions, remaining - self._deps = None - self._rdeps = None - self._prdeps = None - self._provides = None - - @property - def state(self): - m = self.matches_cur - return (len(self.solution_filters), - m.repo, m, - self.matches, - self._deps, - self._rdeps, - self._prdeps) - - @staticmethod - def _filter_choices(cnf_reqs, filterset): - for choices in cnf_reqs: - l = [x for x in choices if x not in filterset] - if not l: - return - yield l - - def _internal_force_next(self): - """ - force next pkg without triggering a reduce_atoms call - @return: True if pkgs remain, False if no more remain - """ - for self.matches_cur in self.matches: - self._reset_iters() - return True - self.matches_cur = self.matches = None - return False - - def reduce_atoms(self, atom): - - if self.matches is None: - raise IndexError("no solutions remain") - if hasattr(atom, "__contains__") and not isinstance(atom, basestring): - self.solution_filters.update(atom) - else: - self.solution_filters.add(atom) - - filterset = self.solution_filters - if self.matches_cur is None: - if not self._internal_force_next(): - return True - - round = -1 - while True: - round += 1 - if round: - if not self._internal_force_next(): - return True - - for depset_name in ("_deps", "_rdeps", "_prdeps"): - depset = getattr(self, depset_name) - reqs = list(self._filter_choices(depset, filterset)) - if len(reqs) != len(depset): - break - setattr(self, depset_name, reqs) - else: - return round > 0 - - def _reset_iters(self): - cur = self.matches_cur - self._deps = cur.depends.cnf_solutions() - self._rdeps = cur.rdepends.cnf_solutions() - self._prdeps = cur.post_rdepends.cnf_solutions() - self._provides = tuple(iter_stable_unique(cur.provides)) - - @property - def slot(self): - return self.current_pkg.slot - - @property - def key(self): - return self.current_pkg.key - - @property - def current_pkg(self): - if self.matches_cur is None: - if self.matches is None: - raise IndexError("no packages remain") - for self.matches_cur in self.matches: - break - else: - self.matches = None - raise IndexError("no more packages remain") - self._reset_iters() - return self.matches_cur - - def force_next_pkg(self): - if self.matches is None: - return False - for self.matches_cur in self.matches: - break - else: - self.matches_cur = self.matches = None - return False - return self.reduce_atoms([]) - - @property - def depends(self): - if not self: - raise IndexError("no more solutions remain") - return self._deps - - @property - def rdepends(self): - if not self: - raise IndexError("no more solutions remain") - return self._rdeps - - @property - def post_rdepends(self): - if not self: - raise IndexError("no more solutions remain") - return self._prdeps - - @property - def provides(self): - if not self: - raise IndexError("no more solutions remain") - return self._provides - - def __nonzero__(self): - if self.matches_cur is None: - if self.matches is None: - return False - for self.matches_cur in self.matches: - break - else: - self.matches = None - return False - self._reset_iters() - return True diff --git a/pkgcore/resolver/pigeonholes.py b/pkgcore/resolver/pigeonholes.py deleted file mode 100644 index 057dc82..0000000 --- a/pkgcore/resolver/pigeonholes.py +++ /dev/null @@ -1,82 +0,0 @@ -# Copyright: 2006-2007 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -from pkgcore.restrictions import restriction - -# lil too getter/setter like for my tastes... - -class PigeonHoledSlots(object): - """class for tracking slotting to a specific atom/obj key - no atoms present, just prevents conflicts of obj.key; atom present, assumes - it's a blocker and ensures no obj matches the atom for that key - """ - - def __init__(self): - self.slot_dict = {} - self.limiters = {} - - def fill_slotting(self, obj, force=False): - """Try to insert obj in. - - @return: any conflicting objs (empty list if inserted successfully). - """ - key = obj.key - l = [x for x in self.limiters.get(key, ()) if x.match(obj)] - - dslot = obj.slot - l.extend(x for x in self.slot_dict.get(key, ()) if x.slot == dslot) - - if not l or force: - self.slot_dict.setdefault(key, []).append(obj) - return l - - - def get_conflicting_slot(self, pkg): - for x in self.slot_dict.get(pkg.key, ()): - if pkg.slot == x.slot: - return x - return None - - def find_atom_matches(self, atom, key=None): - if key is None: - key = atom.key - return filter(atom.match, self.slot_dict.get(key, ())) - - def add_limiter(self, atom, key=None): - """add a limiter, returning any conflicting objs""" - if not isinstance(atom, restriction.base): - raise TypeError("atom must be a restriction.base derivative: " - "got %r, key=%r" % (atom, key)) - # debug. - - if key is None: - key = atom.key - self.limiters.setdefault(key, []).append(atom) - return filter(atom.match, self.slot_dict.get(key, ())) - - def remove_slotting(self, obj): - key = obj.key - # let the key error be thrown if they screwed up. - l = [x for x in self.slot_dict[key] if x is not obj] - if len(l) == len(self.slot_dict[key]): - raise KeyError("obj %s isn't slotted" % obj) - if l: - self.slot_dict[key] = l - else: - del self.slot_dict[key] - - def remove_limiter(self, atom, key=None): - if key is None: - key = atom.key - l = [x for x in self.limiters[key] if x is not atom] - if len(l) == len(self.limiters[key]): - raise KeyError("obj %s isn't slotted" % atom) - if not l: - del self.limiters[key] - else: - self.limiters[key] = l - - def __contains__(self, obj): - if isinstance(obj, restriction.base): - return obj in self.limiters.get(obj.key, ()) - return obj in self.slot_dict.get(obj.key, ()) diff --git a/pkgcore/resolver/plan.py b/pkgcore/resolver/plan.py deleted file mode 100644 index 5edfdd1..0000000 --- a/pkgcore/resolver/plan.py +++ /dev/null @@ -1,877 +0,0 @@ -# Copyright: 2006-2007 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -import operator -from itertools import chain, islice, ifilterfalse -from collections import deque - -from pkgcore.resolver.choice_point import choice_point -from pkgcore.restrictions import packages, values, restriction -from pkgcore.repository.misc import caching_repo -from pkgcore.resolver import state - -from snakeoil.currying import partial, post_curry -from snakeoil.compatibility import any -from snakeoil.iterables import caching_iter, iter_sort - - -limiters = set(["cycle"])#, None]) -def dprint(fmt, args=None, label=None): - if None in limiters or label in limiters: - if args is None: - print fmt - else: - print fmt % args - - -#iter/pkg sorting functions for selection strategy -pkg_sort_highest = partial(sorted, reverse=True) -pkg_sort_lowest = sorted - -pkg_grabber = operator.itemgetter(0) - -def highest_iter_sort(l, pkg_grabber=pkg_grabber): - def f(x, y): - c = cmp(x, y) - if c: - return c - elif x.repo.livefs: - if y.repo.livefs: - return 0 - return 1 - elif y.repo.livefs: - return -1 - return 0 - l.sort(f, key=pkg_grabber, reverse=True) - return l - - -def lowest_iter_sort(l, pkg_grabber=pkg_grabber): - def f(x, y): - c = cmp(x, y) - if c: - return c - elif x.repo.livefs: - if y.repo.livefs: - return 0 - return -1 - elif y.repo.livefs: - return 1 - return 0 - l.sort(f, key=pkg_grabber) - return l - - -class resolver_frame(object): - - __slots__ = ("parent", "atom", "choices", "mode", "start_point", "dbs", - "depth", "drop_cycles", "__weakref__", "ignored", "vdb_limited", - "events", "succeeded") - - def __init__(self, parent, mode, atom, choices, dbs, start_point, depth, - drop_cycles, ignored=False, vdb_limited=False): - self.parent = parent - self.atom = atom - self.choices = choices - self.dbs = dbs - self.mode = mode - self.start_point = start_point - self.depth = depth - self.drop_cycles = drop_cycles - self.ignored = False - self.vdb_limited = vdb_limited - self.events = [] - self.succeeded = None - - def reduce_solutions(self, nodes): - self.events.append(("reduce", nodes)) - return self.choices.reduce_atoms(nodes) - - def __str__(self): - pkg = self.current_pkg - if pkg is None: - pkg = "exhausted" - else: - cpv = pkg.cpvstr - pkg = getattr(pkg.repo, 'repo_id', None) - if pkg is not None: - pkg = "%s::%s" % (cpv, pkg) - else: - pkg = str(pkg) - if self.succeeded is not None: - result = ": %s" % (self.succeeded and "succeeded" or "failed") - else: - result = "" - return "frame%s: mode %r: atom %s: current %s%s%s%s" % \ - (result, self.mode, self.atom, pkg, - self.drop_cycles and ": cycle dropping" or '', - self.ignored and ": ignored" or '', - self.vdb_limited and ": vdb limited" or '') - - @property - def current_pkg(self): - try: - return self.choices.current_pkg - except IndexError: - return None - - -class resolver_stack(deque): - - frame_klass = resolver_frame - depth = property(len) - current_frame = property(operator.itemgetter(-1)) - filter_ignored = staticmethod( - partial(ifilterfalse, operator.attrgetter("ignored"))) - - # this *has* to be a property, else it creates a cycle. - parent = property(lambda s:s) - - def __init__(self): - self.events = [] - - def __str__(self): - return 'resolver stack:\n %s' % '\n '.join(str(x) for x in self) - - def __repr__(self): - return '<%s: %r>' % (self.__class__.__name__, - tuple(repr(x) for x in self)) - - def add_frame(self, mode, atom, choices, dbs, start_point, drop_cycles, vdb_limited=False): - if not self: - parent = self - else: - parent = self[-1] - frame = self.frame_klass(parent, mode, atom, choices, dbs, start_point, - self.depth + 1, drop_cycles, vdb_limited=vdb_limited) - self.append(frame) - return frame - - def add_event(self, event): - if not self: - self.events.append(event) - else: - self[-1].events.append(event) - - def pop_frame(self, result): - frame = self.pop() - frame.succeeded = bool(result) - frame.parent.events.append(frame) - - def will_cycle(self, atom, cur_choice, attr, start=0): - # short cut... - if attr == "post_rdepends": - # not possible for a cycle we'll care about to exist. - # the 'cut off' point is for the new atom, thus not possible for - # a cycle. - return -1 - - cycle_start = -1 - if start != 0: - i = islice(self, start, None) - else: - i = self - for idx, x in enumerate(i): - if x.mode == "post_rdepends": - cycle_start = -1 - if x.atom == atom: - cycle_start = idx - - if cycle_start != -1: - # deque can't be sliced, thus islice - if attr is not None: - s = ', '.join('[%s: %s]' % (x.atom, x.current_pkg) for x in - islice(self, cycle_start)) - if s: - s += ', ' - s += '[%s: %s]' % (atom, cur_choice.current_pkg) - dprint("%s level cycle: stack: %s\n", - (attr, s), "cycle") - return cycle_start + start - return -1 - - def pkg_cycles(self, trg_frame, **kwds): - pkg = trg_frame - return (frame for frame in self._cycles(trg_frame, skip_trg_frame=True, - **kwds) - if pkg == frame.current_pkg) - - def atom_cycles(self, trg_frame, **kwds): - atom = trg_frame.atom - return (frame for frame in self._cycles(trg_frame, skip_trg_frame=True, - **kwds) - if atom == frame.atom) - - def slot_cycles(self, trg_frame, **kwds): - pkg = trg_frame.current_pkg - slot = pkg.slot - key = pkg.key - return (frame for frame in self._cycles(trg_frame, skip_trg_frame=True, - **kwds) - if key == frame.current_pkg.key and slot == frame.current_pkg.slot) - - def _cycles(self, trg_frame, start=0, reverse=False, skip_trg_frame=True): - i = self.filter_ignored(self) - if reverse: - i = self.filter_ignored(reversed(self)) - else: - i = self.filter_ignored(self) - if start != 0: - i = islice(i, start, None) - if skip_trg_frame: - return (frame for frame in i if frame is not trg_frame) - return i - - def index(self, frame, start=0, stop=None): - if start != 0 or stop is not None: - i = slice(self, start, stop) - else: - i = self - for idx, x in enumerate(self): - if x == frame: - return idx + start - return -1 - - -class merge_plan(object): - - vdb_restrict = packages.PackageRestriction("repo.livefs", - values.EqualityMatch(True)) - - def __init__(self, dbs, per_repo_strategy, - global_strategy=None, - depset_reorder_strategy=None, - process_built_depends=False, - drop_cycles=False, debug=False): - - if not isinstance(dbs, (list, tuple)): - dbs = [dbs] - - if global_strategy is None: - global_strategy = self.default_global_strategy - - if depset_reorder_strategy is None: - depset_reorder_strategy = self.default_depset_reorder_strategy - - self.depset_reorder = depset_reorder_strategy - self.per_repo_strategy = per_repo_strategy - self.global_strategy = global_strategy - self.forced_atoms = set() - self.all_dbs = [caching_repo(x, self.per_repo_strategy) for x in dbs] - self.livefs_dbs = [x for x in self.all_dbs if x.livefs] - self.dbs = [x for x in self.all_dbs if not x.livefs] - self.state = state.plan_state() - self.insoluble = set() - self.vdb_preloaded = False - self.drop_cycles = drop_cycles - self.process_built_depends = process_built_depends - if debug: - self._rec_add_atom = partial(self._stack_debugging_rec_add_atom, - self._rec_add_atom) - self._debugging_depth = 0 - self._debugging_drop_cycles = False - - def notify_starting_mode(self, mode, stack): - if mode == "post_rdepends": - mode = 'prdepends' - dprint("%s:%s%s: started: %s" % - (mode, ' ' * ((stack.current_frame.depth * 2) + 12 - len(mode)), - stack.current_frame.atom, - stack.current_frame.choices.current_pkg) - ) - - def notify_trying_choice(self, stack, atom, choices): - dprint("choose for %s%s, %s", - (stack.depth *2*" ", atom, choices.current_pkg)) - stack.add_event(('inspecting', choices.current_pkg)) - - def notify_choice_failed(self, stack, atom, choices, msg, msg_args=()): - if msg: - msg = ': %s' % (msg % msg_args) - dprint("choice for %s%s, %s failed%s", - (stack.depth * 2 * ' ', atom, choices.current_pkg, msg)) -# stack[-1].events.append("failed") - - def notify_choice_succeeded(self, stack, atom, choices, msg='', msg_args=()): - if msg: - msg = ': %s' % (msg % msg_args) - dprint("choice for %s%s, %s succeeded%s", - (stack.depth * 2 * ' ', atom, choices.current_pkg, msg)) -# stack[-1].events.append("succeeded") - - def load_vdb_state(self): - for r in self.livefs_dbs: - for pkg in r.__db__: - dprint("inserting %s from %s", (pkg, r), "vdb") - ret = self.add_atom(pkg.versioned_atom, dbs=self.livefs_dbs) - dprint("insertion of %s from %s: %s", (pkg, r, ret), "vdb") - if ret: - raise Exception( - "couldn't load vdb state, %s %s" % - (pkg.versioned_atom, ret)) - self.vdb_preloaded = True - - def add_atom(self, atom, dbs=None): - """add an atom, recalculating as necessary. - - @return: the last unresolvable atom stack if a solution can't be found, - else returns None if the atom was successfully added. - """ - if dbs is None: - dbs = self.all_dbs - if atom not in self.forced_atoms: - stack = resolver_stack() - ret = self._rec_add_atom(atom, stack, dbs) - if ret: - dprint("failed- %s", ret) - return ret, stack.events[0] - else: - self.forced_atoms.add(atom) - - return () - - def check_for_cycles(self, stack, cur_frame): - """check the current stack for cyclical issues; - @param stack: current stack, a L{resolver_stack} instance - @param cur_frame: current frame, a L{resolver_frame} instance - @return: True if no issues and resolution should continue, else the - value to return after collapsing the calling frame - """ - force_vdb = False - for frame in stack.slot_cycles(cur_frame, reverse=True): - if True: - # exact same pkg. - if frame.mode == 'depends': - # ok, we *must* go vdb if not already. - if frame.current_pkg.repo.livefs: - if cur_frame.current_pkg.repo.livefs: - return None - # force it to vdb. - if cur_frame.current_pkg.repo.livefs: - return True - elif cur_frame.current_pkg == frame.current_pkg and \ - cur_frame.mode == 'post_rdepends': - # if non vdb and it's a post_rdeps cycle for the cur - # node, exempt it; assuming the stack succeeds, - # it's satisfied - return True - force_vdb = True - break - else: - # should be doing a full walk of the cycle here, seeing - # if an rdep becomes a dep. - return None - # portage::gentoo -> rysnc -> portage::vdb; let it process it. - return True - # only need to look at the most recent match; reasoning is simple, - # logic above forces it to vdb if needed. - break - if not force_vdb: - return True - # we already know the current pkg isn't livefs; force livefs to - # sidestep this. - cur_frame.parent.events.append(("cycle", frame, cur_frame, "limiting to vdb")) - cur_frame.ignored = True - return self._rec_add_atom(cur_frame.atom, stack, - self.livefs_dbs, mode=cur_frame.mode, - drop_cycles = cur_frame.drop_cycles) - - - def process_dependencies(self, stack, attr, depset): - failure = [] - additions, blocks, = [], [] - cur_frame = stack.current_frame - self.notify_starting_mode(attr, stack) - for potentials in depset: - failure = [] - for or_node in potentials: - if or_node.blocks: - blocks.append(or_node) - break - failure = self._rec_add_atom(or_node, stack, - cur_frame.dbs, mode=attr, - drop_cycles=cur_frame.drop_cycles) - if failure: - # XXX this is whacky tacky fantastically crappy - # XXX kill it; purpose seems... questionable. - if failure and cur_frame.drop_cycles: - dprint("%s level cycle: %s: " - "dropping cycle for %s from %s", - (attr, cur_frame.atom, datom, - cur_frame.current_pkg), - "cycle") - failure = None - break - - if cur_frame.reduce_solutions(or_node): - # pkg changed. - return [failure] - continue - additions.append(or_node) - break - else: # didn't find any solutions to this or block. - cur_frame.reduce_solutions(potentials) - return [potentials] - else: # all potentials were usable. - return additions, blocks - - def insert_choice(self, atom, stack, choices): - # well, we got ourselvs a resolution. - # do a trick to make the resolver now aware of vdb pkgs if needed - if not self.vdb_preloaded and not choices.current_pkg.repo.livefs: - slotted_atom = choices.current_pkg.slotted_atom - l = self.state.match_atom(slotted_atom) - if not l: - # hmm. ok... no conflicts, so we insert in vdb matches - # to trigger a replace instead of an install - for repo in self.livefs_dbs: - m = repo.match(slotted_atom) - if m: - c = choice_point(slotted_atom, m) - state.add_op(c, c.current_pkg, force=True).apply(self.state) - break - - # first, check for conflicts. - # lil bit fugly, but works for the moment - conflicts = state.add_op(choices, choices.current_pkg).apply(self.state) - if conflicts: - # this means in this branch of resolution, someone slipped - # something in already. cycle, basically. - # hack. see if what was insert is enough for us. - - # this is tricky... if it's the same node inserted - # (cycle), then we ignore it; this does *not* perfectly - # behave though, doesn't discern between repos. - - if (len(conflicts) == 1 and conflicts[0] == choices.current_pkg and - conflicts[0].repo.livefs == choices.current_pkg.repo.livefs and - atom.match(conflicts[0])): - - # early exit. means that a cycle came about, but exact - # same result slipped through. - return False - - dprint("was trying to insert atom '%s' pkg '%s',\n" - "but '[%s]' exists already", - (atom, choices.current_pkg, - ", ".join(map(str, conflicts)))) - - try_rematch = False - if any(True for x in conflicts if isinstance(x, restriction.base)): - # blocker was caught - try_rematch = True - elif not any (True for x in conflicts if not - self.vdb_restrict.match(x)): - # vdb entry, replace. - if self.vdb_restrict.match(choices.current_pkg): - # we're replacing a vdb entry with a vdb entry? wtf. - print ("internal weirdness spotted- vdb restrict matches, " - "but current doesn't, bailing") - raise Exception() - conflicts = state.replace_op(choices, choices.current_pkg).apply( - self.state) - if not conflicts: - dprint("replacing vdb entry for '%s' with pkg '%s'", - (atom, choices.current_pkg)) - - else: - try_rematch = True - if try_rematch: - # XXX: this block looks whacked. figure out what it's up to. - l2 = self.state.match_atom(atom) - if l2 == [choices.current_pkg]: - # stop resolution. - conflicts = False - elif l2: - # potentially need to do some form of cleanup here. - conflicts = False - else: - conflicts = None - return conflicts - - def notify_viable(self, stack, atom, viable, msg='', pre_solved=False): - t_viable = viable and "processing" or "not viable" - if pre_solved and viable: - t_viable = "pre-solved" - t_msg = msg and (" "+msg) or '' - s='' - if stack: - s = " for %s " % (stack[-1].atom) - dprint("%s%s%s%s%s", (t_viable.ljust(13), " "*stack.depth, atom, s, t_msg)) - stack.add_event(("viable", viable, pre_solved, atom, msg)) - - def _viable(self, atom, stack, dbs, limit_to_vdb): - """ - internal function to discern if an atom is viable, returning - the choicepoint/matches iter if viable. - - @return: 3 possible; None (not viable), True (presolved), - L{caching_iter} (not solved, but viable), L{choice_point} - """ - if atom in self.insoluble: - self.notify_viable(stack, atom, False, "globally insoluble") - return None - l = self.state.match_atom(atom) - if l: - self.notify_viable(stack, atom, True, pre_solved=True) - return True - # not in the plan thus far. - matches = caching_iter(self.global_strategy(self, dbs, atom)) - if matches: - choices = choice_point(atom, matches) - # ignore what dropped out, at this juncture we don't care. - choices.reduce_atoms(self.insoluble) - if choices: - return choices, matches - # and was intractable because it has a hard dep on an - # unsolvable atom. - self.notify_viable(stack, atom, False, - msg="pruning of insoluble deps left no choices") - else: - self.notify_viable(stack, atom, False, - msg="no matches") - - if not limit_to_vdb: - self.insoluble.add(atom) - return None - - def insert_blockers(self, stack, choices, blocks): - # level blockers. - fail = True - for x in blocks: - # check for any matches; none, try and insert vdb nodes. - if not self.vdb_preloaded and \ - not choices.current_pkg.repo.livefs and \ - not self.state.match_atom(x): - for repo in self.livefs_dbs: - m = repo.match(x) - if m: - dprint("inserting vdb node for blocker" - " %s %s" % (x, m[0])) - # ignore blockers for for vdb atm, since - # when we level this nodes blockers they'll - # hit - c = choice_point(x, m) - state.add_op(c, c.current_pkg, force=True).apply( - self.state) - break - - rewrote_blocker = self.generate_mangled_blocker(choices, x) - l = self.state.add_blocker(choices, rewrote_blocker, key=x.key) - if l: - # blocker caught something. yay. - dprint("%s blocker %s hit %s for atom %s pkg %s", - (stack[-1].mode, x, l, stack[-1].atom, choices.current_pkg)) - stack.add_event(("blocker", x, l)) - return [x] - return None - - def _stack_debugging_rec_add_atom(self, func, atom, stack, dbs, **kwds): - current = len(stack) - cycles = kwds.get('drop_cycles', False) - reset_cycles = False - if cycles and not self._debugging_drop_cycles: - self._debugging_drop_cycles = reset_cycles = True - if not reset_cycles: - self._debugging_depth += 1 - - assert current == self._debugging_depth -1 - ret = func(atom, stack, dbs, **kwds) - assert current == len(stack) - assert current == self._debugging_depth -1 - if not reset_cycles: - self._debugging_depth -= 1 - else: - self._debugging_drop_cycles = False - return ret - - def _rec_add_atom(self, atom, stack, dbs, mode="none", drop_cycles=False): - """Add an atom. - - @return: False on no issues (inserted succesfully), - else a list of the stack that screwed it up. - """ - limit_to_vdb = dbs == self.livefs_dbs - - depth = stack.depth - - matches = self._viable(atom, stack, dbs, limit_to_vdb) - if matches is None: - return [atom] - elif matches is True: - return None - choices, matches = matches - - if stack: - if limit_to_vdb: - dprint("processing %s%s [%s]; mode %s vdb bound", - (depth*2*" ", atom, stack[-1].atom, mode)) - else: - dprint("processing %s%s [%s]; mode %s", - (depth*2*" ", atom, stack[-1].atom, mode)) - else: - dprint("processing %s%s", (depth*2*" ", atom)) - - stack.add_frame(mode, atom, choices, dbs, - self.state.current_state, drop_cycles, vdb_limited=limit_to_vdb) - ret = self.check_for_cycles(stack, stack.current_frame) - if ret is not True: - stack.pop_frame(ret is None) - return ret - - blocks = [] - failures = [] - - last_state = None - while choices: - new_state = choices.state - if last_state == new_state: - raise AssertionError("no state change detected, " - "old %r != new %r\nchoices(%r)\ncurrent(%r)\ndepends(%r)\n" - "rdepends(%r)\npost_rdepends(%r)\nprovides(%r)" % - (last_state, new_state, tuple(choices.matches), - choices.current_pkg, choices.depends, - choices.rdepends, choices.post_rdepends, - choices.provides)) - last_state = new_state - additions, blocks = [], [] - - self.notify_trying_choice(stack, atom, choices) - - if not choices.current_pkg.built or self.process_built_depends: - l = self.process_dependencies(stack, "depends", - self.depset_reorder(self, choices.depends, "depends")) - if len(l) == 1: - dprint("reseting for %s%s because of depends: %s", - (depth*2*" ", atom, l[0][-1])) - self.state.backtrack(stack.current_frame.start_point) - failures = l[0] - continue - additions += l[0] - blocks = l[1] - - # level blockers. - ret = self.insert_blockers(stack, choices, blocks) - if ret is not None: - # hackish in terms of failures, needs cleanup - failures = ret - self.notify_choice_failed(stack, atom, choices, - "failed due to %s", (ret,)) - stack.current_frame.reduce_solutions(ret) - self.state.backtrack(stack.current_frame.start_point) - continue - - l = self.process_dependencies(stack, "rdepends", - self.depset_reorder(self, choices.rdepends, "rdepends")) - if len(l) == 1: - dprint("reseting for %s%s because of rdepends: %s", - (depth*2*" ", atom, l[0])) - self.state.backtrack(stack.current_frame.start_point) - failures = l[0] - continue - additions += l[0] - blocks = l[1] - - l = self.insert_choice(atom, stack, choices) - if l is False: - # this means somehow the node already slipped in. - # so we exit now, we are satisfied - self.notify_choice_succeeded(stack, atom, choices, - "already exists in the state plan") - stack.pop_frame(True) - return None - elif l is not None: - # failure. - self.notify_choice_failed(stack, atom, choices, - "failed inserting: %s", l) - self.state.backtrack(stack.current_frame.start_point) - choices.force_next_pkg() - continue - - # XXX: push this into a method. - fail = True - for x in choices.provides: - l = state.add_op(choices, x).apply(self.state) - if l and l != [x]: - # slight hack; basically, should be pruning providers as the parent is removed - # this duplicates it, basically; if it's not a restrict, then it's a pkg. - # thus poke it. - if len(l) == 1 and not isinstance(l[0], restriction.base): - p = getattr(l[0], 'provider', None) - if p is not None and not self.state.match_atom(p): - # ok... force it. - fail = state.replace_op(choices, x).apply(self.state) - if not fail: - continue - break - fail = l - break - else: - fail = False - if fail: - self.state.backtrack(stack.current_frame.start_point) - choices.force_next_pkg() - continue - - ret = self.insert_blockers(stack, choices, blocks) - if ret is not None: - # hackish in terms of failures, needs cleanup - failures = ret - self.notify_choice_failed(stack, atom, choices, - "failed due to %s", (ret,)) - stack.current_frame.reduce_solutions(ret) - self.state.backtrack(stack.current_frame.start_point) - continue - - l = self.process_dependencies(stack, "post_rdepends", - self.depset_reorder(self, choices.post_rdepends, - "post_rdepends")) - - if len(l) == 1: - dprint("resetting for %s%s because of rdepends: %s", - (depth*2*" ", atom, l[0])) - self.state.backtrack(stack.current_frame.start_point) - failures = l[0] - continue - additions += l[0] - blocks = l[1] - - # level blockers. - ret = self.insert_blockers(stack, choices, blocks) - if ret is not None: - # hackish in terms of failures, needs cleanup - failures = ret - self.notify_choice_failed(stack, atom, choices, - "failed due to %s", (ret,)) - stack.current_frame.reduce_solutions(ret) - self.state.backtrack(stack.current_frame.start_point) - continue - # kinky... the request is fully satisfied - break - - else: - dprint("no solution %s%s", (depth*2*" ", atom)) - self.state.backtrack(stack.current_frame.start_point) - # saving roll. if we're allowed to drop cycles, try it again. - # this needs to be *far* more fine grained also. it'll try - # regardless of if it's cycle issue - if not drop_cycles and self.drop_cycles: - dprint("trying saving throw for %s ignoring cycles", - atom, "cycle") - # note everything is retored to a pristine state prior also. - stack[-1].ignored = True - l = self._rec_add_atom(atom, stack, dbs, - mode=mode, drop_cycles=True) - if not l: - stack.pop_frame(True) - return None - stack.pop_frame(False) - return [atom] + failures - - self.notify_choice_succeeded(stack, atom, choices) - stack.pop_frame(True) - return None - - def generate_mangled_blocker(self, choices, blocker): - """converts a blocker into a "cannot block ourself" block""" - # note the second Or clause is a bit loose; allows any version to - # slip through instead of blocking everything that isn't the - # parent pkg - if blocker.category != 'virtual': - return blocker - return packages.AndRestriction(blocker, - packages.PackageRestriction("provider.key", - values.StrExactMatch(choices.current_pkg.key), - negate=True, ignore_missing=True), - finalize=True) - - def free_caches(self): - for repo in self.all_dbs: - repo.clear() - - # selection strategies for atom matches - - @staticmethod - def default_depset_reorder_strategy(self, depset, mode): - for or_block in depset: - vdb = [] - non_vdb = [] - if len(or_block) == 1: - yield or_block - continue - for atom in or_block: - if atom.blocks: - non_vdb.append(atom) - elif self.state.match_atom(atom): - vdb.append(atom) - elif caching_iter(p for r in self.livefs_dbs - for p in r.match(atom)): - vdb.append(atom) - else: - non_vdb.append(atom) - if vdb: - yield vdb + non_vdb - else: - yield or_block - - @staticmethod - def default_global_strategy(self, dbs, atom): - return chain(*[repo.match(atom) for repo in dbs]) - - @staticmethod - def just_livefs_dbs(dbs): - return (r for r in dbs if r.livefs) - - @staticmethod - def just_nonlivefs_dbs(dbs): - return (r for r in dbs if not r.livefs) - - @classmethod - def prefer_livefs_dbs(cls, dbs, just_vdb=None): - """ - @param dbs: db list to walk - @param just_vdb: if None, no filtering; if True, just vdb, if False, - non-vdb only - @return: yields repositories in requested ordering - """ - return chain(cls.just_livefs_dbs(dbs), cls.just_nonlivefs_dbs(dbs)) - - @staticmethod - def prefer_highest_version_strategy(self, dbs, atom): - # XXX rework caching_iter so that it iter's properly - return iter_sort(highest_iter_sort, - *[repo.match(atom) - for repo in self.prefer_livefs_dbs(dbs)]) - - @staticmethod - def prefer_lowest_version_strategy(self, dbs, atom): - return iter_sort(lowest_iter_sort, - self.default_global_strategy(self, dbs, atom)) - - @staticmethod - def prefer_reuse_strategy(self, dbs, atom): - - return chain( - iter_sort(highest_iter_sort, - *[repo.match(atom) for repo in self.just_livefs_dbs(dbs)]), - iter_sort(highest_iter_sort, - *[repo.match(atom) for repo in self.just_nonlivefs_dbs(dbs)]) - ) - - def generic_force_version_strategy(self, vdb, dbs, atom, iter_sorter, - pkg_sorter): - try: - # nasty, but works. - yield iter_sort(iter_sorter, - *[r.itermatch(atom, sorter=pkg_sorter) - for r in [vdb] + dbs]).next() - except StopIteration: - # twas no matches - pass - - force_max_version_strategy = staticmethod( - post_curry(generic_force_version_strategy, - highest_iter_sort, pkg_sort_highest)) - force_min_version_strategy = staticmethod( - post_curry(generic_force_version_strategy, - lowest_iter_sort, pkg_sort_lowest)) diff --git a/pkgcore/resolver/state.py b/pkgcore/resolver/state.py deleted file mode 100644 index 82e19ed..0000000 --- a/pkgcore/resolver/state.py +++ /dev/null @@ -1,205 +0,0 @@ -# Copyright: 2006-2007 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -from snakeoil.containers import RefCountingSet -from pkgcore.resolver.pigeonholes import PigeonHoledSlots - -REMOVE = 0 -ADD = 1 -REPLACE = 2 -FORWARD_BLOCK_INCREF = 3 -FORWARD_BLOCK_DECREF = 4 - -class plan_state(object): - def __init__(self): - self.state = PigeonHoledSlots() - self.plan = [] - self.pkg_choices = {} - self.rev_blockers = {} - self.blockers_refcnt = RefCountingSet() - self.match_atom = self.state.find_atom_matches - - def add_blocker(self, choices, blocker, key=None): - """adds blocker, returning any packages blocked""" - return incref_forward_block_op(choices, blocker, key).apply(self) - - def _remove_pkg_blockers(self, choices): - l = self.rev_blockers.get(choices, ()) - # walk a copy- it's possible it'll change under foot - for blocker, key in l[:]: - decref_forward_block_op(choices, blocker, key).apply(self) - - def backtrack(self, state_pos): - assert state_pos <= len(self.plan) - if len(self.plan) == state_pos: - return - for change in reversed(self.plan[state_pos:]): - change.revert(self) - self.plan = self.plan[:state_pos] - - def iter_ops(self, return_livefs=False): - iterable = (x for x in self.plan if not x.internal) - if return_livefs: - return iterable - return (y for y in iterable - if not y.pkg.repo.livefs) - - @property - def current_state(self): - return len(self.plan) - - -class base_op(object): - __slots__ = ("pkg", "force", "choices") - internal = False - - def __init__(self, choices, pkg, force=False): - self.choices = choices - self.pkg = pkg - self.force = force - - def __str__(self): - s = '' - if self.force: - s = ' forced' - return "%s: %s%s" % (self.desc, self.pkg, s) - - -class add_op(base_op): - - desc = "add" - - def apply(self, plan): - l = plan.state.fill_slotting(self.pkg, force=self.force) - if l and not self.force: - return l - plan.pkg_choices[self.pkg] = self.choices - plan.plan.append(self) - - def revert(self, plan): - plan.state.remove_slotting(self.pkg) - del plan.pkg_choices[self.pkg] - - -class remove_op(base_op): - __slots__ = () - - desc = "remove" - - def apply(self, plan): - plan.state.remove_slotting(self.pkg) - plan._remove_pkg_blockers(plan.pkg_choices) - del plan.pkg_choices[self.pkg] - plan.plan.append(self) - - def revert(self, plan): - plan.state.fill_slotting(self.pkg, force=self.force) - plan.pkg_choices[self.pkg] = self.choices - - -class replace_op(base_op): - __slots__ = ("old_pkg", "old_choices") - - desc = "replace" - - def apply(self, plan): - old = plan.state.get_conflicting_slot(self.pkg) - # probably should just convert to an add... - assert old is not None - plan.state.remove_slotting(old) - old_choices = plan.pkg_choices[old] - revert_point = plan.current_state - plan._remove_pkg_blockers(old_choices) - l = plan.state.fill_slotting(self.pkg, force=self.force) - if l: - # revert... limiter. - l2 = plan.state.fill_slotting(old) - plan.backtrack(revert_point) - assert not l2 - return l - - # wipe olds blockers. - - self.old_pkg = old - self.old_choices = old_choices - del plan.pkg_choices[old] - plan.pkg_choices[self.pkg] = self.choices - plan.plan.append(self) - - def revert(self, plan): - # far simpler, since the apply op generates multiple ops on it's own. - # all we have to care about is swap. - plan.state.remove_slotting(self.pkg) - l = plan.state.fill_slotting(self.old_pkg, force=self.force) - assert not l - del plan.pkg_choices[self.pkg] - plan.pkg_choices[self.old_pkg] = self.old_choices - - def __str__(self): - s = '' - if self.force: - s = ' forced' - return "replace: %s with %s%s" % (self.old_pkg, self.pkg, s) - - -class blocker_base_op(object): - __slots__ = ("choices", "blocker", "key") - - desc = None - internal = True - - def __init__(self, choices, blocker, key=None): - if key is None: - self.key = blocker.key - else: - self.key = key - self.choices = choices - self.blocker = blocker - - def __str__(self): - return "%s: key %s, %s from %s" % (self.__class__.__name__, self.key, - self.blocker, self.choices) - - -class incref_forward_block_op(blocker_base_op): - __slots__ = () - - def apply(self, plan): - plan.plan.append(self) - if self.blocker not in plan.blockers_refcnt: - l = plan.state.add_limiter(self.blocker, self.key) - else: - l = [] - plan.rev_blockers.setdefault(self.choices, []).append( - (self.blocker, self.key)) - plan.blockers_refcnt.add(self.blocker) - return l - - def revert(self, plan): - l = plan.rev_blockers[self.choices] - l.remove((self.blocker, self.key)) - if not l: - del plan.rev_blockers[self.choices] - plan.blockers_refcnt.remove(self.blocker) - if self.blocker not in plan.blockers_refcnt: - plan.state.remove_limiter(self.blocker, self.key) - - -class decref_forward_block_op(blocker_base_op): - __slots__ = () - - def apply(self, plan): - plan.plan.append(self) - plan.blockers_refcnt.remove(self.blocker) - if self.blocker not in plan.blockers_refcnt: - plan.state.remove_limiter(self.blocker, self.key) - plan.rev_blockers[self.choices].remove((self.blocker, self.key)) - if not plan.rev_blockers[self.choices]: - del plan.rev_blockers[self.choices] - - def revert(self, plan): - plan.rev_blockers.setdefault(self.choices, []).append( - (self.blocker, self.key)) - if self.blocker not in plan.blockers_refcnt: - plan.state.add_limiter(self.blocker, self.key) - plan.blockers_refcnt.add(self.blocker) diff --git a/pkgcore/resolver/util.py b/pkgcore/resolver/util.py deleted file mode 100644 index 368096e..0000000 --- a/pkgcore/resolver/util.py +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright: 2007 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -from itertools import groupby - -def group_attempts(sequence, filter_func=lambda x:True): - last, l = None, [] - for x in sequence: - if isinstance(x, tuple) and x[0] == 'inspecting': - if l: - yield last, l - last, l = x[1], [] - elif last is not None: - if filter_func(x): - # inline ignored frames - if getattr(x, 'ignored', False): - l.extend(y for y in x.events if filter_func(y)) - else: - l.append(x) - if l: - yield last, l - -def fails_filter(x): - if not isinstance(x, tuple): - return not x.succeeded - if x[0] == "viable": - return not x[1] - return x[0] != "inspecting" - -def reduce_to_failures(frame): - if frame.succeeded: - return [] - l = [frame] - for pkg, nodes in group_attempts(frame.events, fails_filter): - l2 = [] - for x in nodes: - if not isinstance(x, tuple): - l2.append(reduce_to_failures(x)) - else: - l2.append(x) - l.append((pkg, l2)) - return l diff --git a/pkgcore/restrictions/__init__.py b/pkgcore/restrictions/__init__.py deleted file mode 100644 index dff841e..0000000 --- a/pkgcore/restrictions/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -# Copyright: 2005 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -"""restriction subsystem, used both for depencies and general package queries""" diff --git a/pkgcore/restrictions/boolean.py b/pkgcore/restrictions/boolean.py deleted file mode 100644 index 0552c40..0000000 --- a/pkgcore/restrictions/boolean.py +++ /dev/null @@ -1,490 +0,0 @@ -# Copyright: 2005-2007 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -"""Boolean combinations of restrictions. - -This module provides classes that can be used to combine arbitrary -collections of restrictions in AND, NAND, OR, NOR, XOR, XNOR style -operations. -""" - -__all__ = ("AndRestriction", "OrRestriction") - -from itertools import islice -from pkgcore.restrictions import restriction -from snakeoil.klass import generic_equality - -class base(restriction.base): - - """base template for boolean restrictions""" - - __metaclass__ = generic_equality - __attr_comparison__ = ('negate', 'type', 'restrictions') - __slots__ = ('restrictions', 'type', 'negate') - - def __init__(self, *restrictions, **kwds): - - """ - @keyword node_type: type of restriction this accepts - (L{package_type<pkgcore.restrictions.packages.package_type>} and - L{value_type<pkgcore.restrictions.values.value_type>} being - common types). If set to C{None}, no instance limiting is done. - @type restrictions: node_type (if that is specified) - @param restrictions: initial restrictions to add - @keyword finalize: should this instance be made immutable immediately? - @keyword negate: should the logic be negated? - """ - - sf = object.__setattr__ - - node_type = kwds.pop("node_type", None) - - sf(self, "type", node_type) - sf(self, "negate", kwds.pop("negate", False)) - - if node_type is not None: - try: - for r in restrictions: - if r.type is not None and r.type != node_type: - raise TypeError( - "instance '%s' is restriction type '%s', " - "must be '%s'" % (r, r.type, node_type)) - except AttributeError: - raise TypeError( - "type '%s' instance '%s' has no restriction type, " - "'%s' required" % ( - r.__class__, r, node_type)) - - if kwds.pop("finalize", False): - if not isinstance(restrictions, tuple): - sf(self, "restrictions", tuple(restrictions)) - else: - sf(self, "restrictions", restrictions) - else: - sf(self, "restrictions", list(restrictions)) - - if kwds: - kwds.pop("disable_inst_caching", None) - if kwds: - raise TypeError("unknown keywords to %s: %s" % - (self.__class__, kwds)) - - def change_restrictions(self, *restrictions, **kwds): - """ - return a new instance of self.__class__, using supplied restrictions - - """ - if self.type is not None: - if self.__class__.type not in restriction.valid_types or \ - self.__class__.type != self.type: - kwds["node_type"] = self.type - kwds.setdefault("negate", self.negate) - return self.__class__(*restrictions, **kwds) - - def add_restriction(self, *new_restrictions): - """ - add an more restriction(s) - - @param new_restrictions: if node_type is enforced, - restrictions must be of that type. - """ - - if not new_restrictions: - raise TypeError("need at least one restriction handed in") - if self.type is not None: - try: - for r in new_restrictions: - if r.type is not None and r.type != self.type: - raise TypeError( - "instance '%s' is restriction type '%s', " - "must be '%s'" % (r, r.type, self.type)) - except AttributeError: - raise TypeError( - "type '%s' instance '%s' has no restriction type, " - "'%s' required" % ( - r.__class__, r, getattr(self, "type", "unset"))) - - try: - self.restrictions.extend(new_restrictions) - except AttributeError: - raise TypeError("%r is finalized" % self) - - def finalize(self): - """ - finalize the restriction instance, disallowing adding restrictions. - """ - object.__setattr__(self, "restrictions", tuple(self.restrictions)) - - def __repr__(self): - return '<%s negate=%r type=%r finalized=%r restrictions=%r @%#8x>' % ( - self.__class__.__name__, self.negate, getattr(self, 'type', None), - isinstance(self.restrictions, tuple), self.restrictions, - id(self)) - - def __len__(self): - return len(self.restrictions) - - def __iter__(self): - return iter(self.restrictions) - - def match(self, action, *vals): - raise NotImplementedError - - force_False, force_True = match, match - - def dnf_solutions(self, full_solution_expansion=False): - raise NotImplementedError() - - cnf_solutions = dnf_solutions - - def iter_cnf_solutions(self, *a, **kwds): - """iterate over the cnf solution""" - return iter(self.cnf_solutions(*a, **kwds)) - - def iter_dnf_solutions(self, *a, **kwds): - """iterate over the dnf solution""" - return iter(self.dnf_solutions(*a, **kwds)) - - def __getitem__(self, key): - return self.restrictions[key] - - -# this beast, handles N^2 permutations. convert to stack based. -def iterative_quad_toggling(pkg, pvals, restrictions, starting, end, truths, - filter, desired_false=None, desired_true=None, - kill_switch=None): - if desired_false is None: - desired_false = lambda r, a:r.force_False(*a) - if desired_true is None: - desired_true = lambda r, a:r.force_True(*a) - -# import pdb;pdb.set_trace() - reset = True - if starting == 0: - if filter(truths): - yield True - for index, rest in islice(enumerate(restrictions), starting, end): - if reset: - entry = pkg.changes_count() - reset = False - if truths[index]: - if desired_false(rest, pvals): - reset = True - t = truths[:] - t[index] = False - if filter(t): - yield True - for i in iterative_quad_toggling( - pkg, pvals, restrictions, index + 1, end, t, filter, - desired_false=desired_false, desired_true=desired_true, - kill_switch=kill_switch): -# import pdb;pdb.set_trace() - yield True - reset = True - else: - if kill_switch is not None and kill_switch(truths, index): - return - else: - if desired_true(rest, pvals): - reset = True - t = truths[:] - t[index] = True - if filter(t): - yield True - for x in iterative_quad_toggling( - pkg, pvals, restrictions, index + 1, end, t, filter, - desired_false=desired_false, desired_true=desired_true): -# import pdb;pdb.set_trace() - yield True - reset = True - elif index == end: - if filter(truths): -# import pdb;pdb.set_trace() - yield True - else: - if kill_switch is not None and kill_switch(truths, index): - return - - if reset: - pkg.rollback(entry) - - -class AndRestriction(base): - """Boolean AND grouping of restrictions. negation is a NAND""" - __slots__ = () - - def match(self, vals): - for rest in self.restrictions: - if not rest.match(vals): - return self.negate - return not self.negate - - def force_True(self, pkg, *vals): - pvals = [pkg] - pvals.extend(vals) - entry_point = pkg.changes_count() - # get the simple one out of the way first. - if not self.negate: - for r in self.restrictions: - if not r.force_True(*pvals): - pkg.rollback(entry_point) - return False - return True - - # <insert page long curse here>, NAND logic, - # len(restrictions)**2 potential solutions. - # 0|0 == 0, 0|1 == 1|0 == 0|0 == 1. - # XXX this is quadratic. patches welcome to dodge the - # requirement to push through all potential truths. - truths = [r.match(*pvals) for r in self.restrictions] - def filter(truths): - return False in truths - - for i in iterative_quad_toggling(pkg, pvals, self.restrictions, 0, - len(self.restrictions), truths, - filter): - return True - return False - - def force_False(self, pkg, *vals): - pvals = [pkg] - pvals.extend(vals) - entry_point = pkg.changes_count() - # get the simple one out of the way first. - if self.negate: - for r in self.restrictions: - if not r.force_True(*pvals): - pkg.rollback(entry_point) - return False - return True - - # <insert page long curse here>, NAND logic, - # (len(restrictions)^2)-1 potential solutions. - # 1|1 == 0, 0|1 == 1|0 == 0|0 == 1. - # XXX this is quadratic. patches welcome to dodge the - # requirement to push through all potential truths. - truths = [r.match(*pvals) for r in self.restrictions] - def filter(truths): - return False in truths - for i in iterative_quad_toggling(pkg, pvals, self.restrictions, 0, - len(self.restrictions), truths, - filter): - return True - return False - - def iter_dnf_solutions(self, full_solution_expansion=False): - """ - generater yielding DNF (disjunctive normalized form) of this instance. - - @param full_solution_expansion: controls whether to expand everything - (break apart atoms for example); this isn't likely what you want - """ - if self.negate: -# raise NotImplementedError("negation for dnf_solutions on " -# "AndRestriction isn't implemented yet") - # hack- this is an experiment - for r in OrRestriction( - node_type=self.type, - *[restriction.Negate(x) - for x in self.restrictions]).iter_dnf_solutions(): - yield r - return - if not self.restrictions: - yield [] - return - hardreqs = [] - optionals = [] - for x in self.restrictions: - if isinstance(x, base): - s2 = x.dnf_solutions( - full_solution_expansion=full_solution_expansion) - assert s2 - if len(s2) == 1: - hardreqs.extend(s2[0]) - else: - optionals.append(s2) - else: - hardreqs.append(x) - def f(arg, *others): - if others: - for node in arg: - for node2 in f(*others): - yield node + node2 - else: - for node in arg: - yield node - - for solution in f([hardreqs], *optionals): - assert isinstance(solution, (tuple, list)) - yield solution - - def dnf_solutions(self, *args, **kwds): - """ - list form of L{iter_dnf_solutions}, see iter_dnf_solutions for args - """ - return list(self.iter_dnf_solutions(*args, **kwds)) - - def cnf_solutions(self, full_solution_expansion=False): - - """ - returns solutions in CNF (conjunctive normalized form) of this instance - - @param full_solution_expansion: controls whether to expand everything - (break apart atoms for example); this isn't likely what you want - """ - - if self.negate: - raise NotImplementedError("negation for solutions on " - "AndRestriction isn't implemented yet") - andreqs = [] - for x in self.restrictions: - if isinstance(x, base): - andreqs.extend(x.cnf_solutions( - full_solution_expansion=full_solution_expansion)) - else: - andreqs.append([x]) - return andreqs - - def __str__(self): - if self.negate: - return "not ( %s )" % " && ".join(str(x) for x in self.restrictions) - return "( %s )" % " && ".join(str(x) for x in self.restrictions) - - -class OrRestriction(base): - """Boolean OR grouping of restrictions.""" - __slots__ = () - - def match(self, vals): - for rest in self.restrictions: - if rest.match(vals): - return not self.negate - return self.negate - - def cnf_solutions(self, full_solution_expansion=False): - """ - returns alist in CNF (conjunctive normalized form) for of this instance - - @param full_solution_expansion: controls whether to expand everything - (break apart atoms for example); this isn't likely what you want - """ - if self.negate: - raise NotImplementedError( - "OrRestriction.solutions doesn't yet support self.negate") - - if not self.restrictions: - return [] - - dcnf = [] - cnf = [] - for x in self.restrictions: - if isinstance(x, base): - s2 = x.dnf_solutions( - full_solution_expansion=full_solution_expansion) - if len(s2) == 1: - cnf.extend(s2) - else: - for y in s2: - if len(y) == 1: - dcnf.append(y[0]) - else: - cnf.append(y) - else: - dcnf.append(x) - - # combinatorial explosion. if it's got cnf, we peel off one of - # each and smash append to the dcnf. - dcnf = [dcnf] - for andreq in cnf: - dcnf = list(y + [x] for x in andreq for y in dcnf) - return dcnf - - - def iter_dnf_solutions(self, full_solution_expansion=False): - """ - returns a list in DNF (disjunctive normalized form) for of this instance - - @param full_solution_expansion: controls whether to expand everything - (break apart atoms for example); this isn't likely what you want - """ - if self.negate: - # hack- this is an experiment - for x in AndRestriction( - node_type=self.type, - *[restriction.Negate(x) - for x in self.restrictions]).iter_dnf_solutions(): - yield x - if not self.restrictions: - yield [] - return - for x in self.restrictions: - if isinstance(x, base): - for y in x.iter_dnf_solutions( - full_solution_expansion=full_solution_expansion): - yield y - else: - yield [x] - - def dnf_solutions(self, *args, **kwds): - """ - see dnf_solutions, iterates yielding DNF solutions - """ - return list(self.iter_dnf_solutions(*args, **kwds)) - - def force_True(self, pkg, *vals): - pvals = [pkg] - pvals.extend(vals) - entry_point = pkg.changes_count() - # get the simple one out of the way first. - if self.negate: - for r in self.restrictions: - if not r.force_False(*pvals): - pkg.rollback(entry_point) - return False - return True - - # <insert page long curse here>, OR logic, - # len(restrictions)**2-1 potential solutions. - # 0|0 == 0, 0|1 == 1|0 == 1|1 == 1. - # XXX this is quadratic. patches welcome to dodge the - # requirement to push through all potential truths. - truths = [r.match(*pvals) for r in self.restrictions] - def filter(truths): - return True in truths - for i in iterative_quad_toggling(pkg, pvals, self.restrictions, 0, - len(self.restrictions), truths, - filter): - return True - return False - - def force_False(self, pkg, *vals): - pvals = [pkg] - pvals.extend(vals) - entry_point = pkg.changes_count() - # get the simple one out of the way first. - if not self.negate: - for r in self.restrictions: - if not r.force_False(*pvals): - pkg.rollback(entry_point) - return - yield True - return - - # <insert page long curse here>, OR logic, - # (len(restrictions)**2)-1 potential solutions. - # 0|0 == 0, 0|1 == 1|0 == 1|1 == 1. - # XXX this is quadratic. patches welcome to dodge the - # requirement to push through all potential truths. - truths = [r.match(*pvals) for r in self.restrictions] - def filter(truths): - return True in truths - for i in iterative_quad_toggling(pkg, pvals, self.restrictions, 0, - len(self.restrictions), truths, - filter): - yield True - - - def __str__(self): - if self.negate: - return "not ( %s )" % " || ".join(str(x) for x in self.restrictions) - return "( %s )" % " || ".join(str(x) for x in self.restrictions) diff --git a/pkgcore/restrictions/delegated.py b/pkgcore/restrictions/delegated.py deleted file mode 100644 index 2d66840..0000000 --- a/pkgcore/restrictions/delegated.py +++ /dev/null @@ -1,53 +0,0 @@ -# Copyright: 2005-2007 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -""" -functionality to hand off to a callable, enabling collapsing -long chains of restrictions into Nlog N, or generating -restrictions on the fly -""" - -__all__ = ("delegate",) -from pkgcore.restrictions import restriction -from pkgcore.restrictions import packages - -class delegate(restriction.base): - - """ - hand off matching to a handed in prototype - - Example usage of this class should be available in - L{pkgcore.ebuild.domain}. - """ - - __slots__ = ('_transform', 'negate') - - type = packages.package_type - inst_caching = False - - def __init__(self, transform_func, negate=False): - """ - - @param transform_func: callable inovked with data, pkg, and mode - mode may be "match", "force_True", or "force_False" - """ - - if not callable(transform_func): - raise TypeError(transform_func) - - object.__setattr__(self, "negate", negate) - object.__setattr__(self, "_transform", transform_func) - - - def match(self, pkginst): - return self._transform(pkginst, "match") != self.negate - - def force_True(self, pkginst): - if self.negate: - return self._transform(pkginst, "force_False") - return self._transform(pkginst, "force_True") - - def force_False(self, pkginst): - if self.negate: - return self._transform(pkginst, "force_True") - return self._transform(pkginst, "force_False") diff --git a/pkgcore/restrictions/packages.py b/pkgcore/restrictions/packages.py deleted file mode 100644 index 620e825..0000000 --- a/pkgcore/restrictions/packages.py +++ /dev/null @@ -1,245 +0,0 @@ -# Copyright: 2005-2007 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -""" -restriction classes designed for package level matching -""" - -from pkgcore.restrictions import restriction, boolean -from snakeoil.compatibility import any -from snakeoil.klass import chained_getter, generic_equality -from snakeoil.demandload import demandload -demandload(globals(), "pkgcore.log:logger") - -# Backwards compatibility. -package_type = restriction.package_type - - -class native_PackageRestriction(object): - __slots__ = ('_pull_attr', 'attr', 'restriction', 'ignore_missing', - 'negate') - - __attr_comparison__ = ("__class__", "negate", "attr", "restriction") - __metaclass__ = generic_equality - - def __init__(self, attr, childrestriction, negate=False, - ignore_missing=True): - """ - @param attr: package attribute to match against - @param childrestriction: a L{pkgcore.restrictions.values.base} instance - to pass attr to for matching - @param negate: should the results be negated? - """ - if not childrestriction.type == self.subtype: - raise TypeError("restriction must be of type %r" % (self.subtype,)) - sf = object.__setattr__ - sf(self, "negate", negate) - sf(self, "_pull_attr", chained_getter(attr)) - sf(self, "attr", attr) - sf(self, "restriction", childrestriction) - sf(self, "ignore_missing", ignore_missing) - - -class PackageRestriction_mixin(restriction.base): - """Package data restriction.""" - - __slots__ = () - - # Careful: some methods (__eq__, __hash__, intersect) try to work - # for subclasses too. They will not behave as intended if a - # subclass adds attributes. So if you do that, override the - # methods. - - type = restriction.package_type - subtype = restriction.value_type - - def _handle_exception(self, pkg, exc): - if isinstance(exc, AttributeError): - if not self.ignore_missing: - logger.exception("failed getting attribute %s from %s, " - "exception %s" % (self.attr, str(pkg), str(exc))) - s = self.attr.split('.') - eargs = [x for x in exc.args if isinstance(x, basestring)] - if any(x in s for x in eargs): - return False - elif any("'%s'" % x in y for x in s for y in eargs): - # this is fairly horrible; probably specific to cpython also. - # either way, does a lookup specifically for attr components - # in the string exception string, looking for 'attr' in the - # text. - # if it doesn't match, exception is thrown. - return False - logger.exception("caught unexpected exception accessing %s from %s, " - "exception %s" % (self.attr, str(pkg), str(exc))) - return True - - def match(self, pkg): - try: - return self.restriction.match(self._pull_attr(pkg)) != self.negate - except (KeyboardInterrupt, RuntimeError, SystemExit): - raise - except Exception, e: - if self._handle_exception(pkg, e): - raise - return self.negate - - def force_False(self, pkg): - try: - if self.negate: - return self.restriction.force_True(pkg, self.attr, - self._pull_attr(pkg)) - else: - return self.restriction.force_False(pkg, self.attr, - self._pull_attr(pkg)) - except (KeyboardInterrupt, RuntimeError, SystemExit): - raise - except Exception, e: - if self._handle_exception(pkg, e): - raise - return not self.negate - - def force_True(self, pkg): - try: - if self.negate: - return self.restriction.force_False(pkg, self.attr, - self._pull_attr(pkg)) - else: - return self.restriction.force_True(pkg, self.attr, - self._pull_attr(pkg)) - except (KeyboardInterrupt, RuntimeError, SystemExit): - raise - except Exception, e: - if self._handle_exception(pkg, e): - raise - return self.negate - - def __len__(self): - if not isinstance(self.restriction, boolean.base): - return 1 - return len(self.restriction) + 1 - - def intersect(self, other): - """Build a restriction that matches anything matched by this and other. - - If an optimized intersection cannot be determined this returns C{None}. - """ - if (self.negate != other.negate or - self.attr != other.attr or - self.__class__ is not other.__class__): - return None - # Make the most subclassed instance do the intersecting - if isinstance(self.restriction, other.restriction.__class__): - s = self.restriction.intersect(other.restriction) - elif isinstance(other.restriction, self.restriction.__class__): - s = other.restriction.intersect(self.restriction) - else: - # Child restrictions are not related, give up. - return None - if s is None: - return None - - # optimization: do not build a new wrapper if we already have one. - if s == self.restriction: - return self - elif s == other.restriction: - return other - - # This breaks completely if we are a subclass with different - # __init__ args, so such a subclass had better override this - # method... - return self.__class__(self.attr, s, negate=self.negate) - - def __hash__(self): - return hash((self.negate, self.attr, self.restriction)) - - def __str__(self): - s = self.attr+" " - if self.negate: - s += "not " - return s + str(self.restriction) - - def __repr__(self): - if self.negate: - string = '<%s attr=%r restriction=%r negated @%#8x>' - else: - string = '<%s attr=%r restriction=%r @%#8x>' - return string % ( - self.__class__.__name__, self.attr, self.restriction, id(self)) - - -try: - from pkgcore.restrictions._restrictions import PackageRestriction as \ - PackageRestriction_base -except ImportError: - PackageRestriction_base = native_PackageRestriction - -class PackageRestriction(PackageRestriction_mixin, PackageRestriction_base): - __slots__ = () - __inst_caching__ = True - -class Conditional(PackageRestriction): - - """ - base object representing a conditional package restriction - - used to control whether a payload of restrictions are accessible or not - """ - - __slots__ = ('payload',) - - __attr_comparison__ = ("__class__", "negate", "attr", "restriction", - "payload") - __metaclass__ = generic_equality - # note that instance caching is turned off. - # rarely pays off for conditionals from a speed/mem comparison - - def __init__(self, attr, childrestriction, payload, **kwds): - """ - @param attr: attr to match against - @param childrestriction: restriction to control whether or not the - payload is accessible - @param payload: payload data, whatever it may be. - @param kwds: additional args to pass to L{PackageRestriction} - """ - PackageRestriction.__init__(self, attr, childrestriction, **kwds) - object.__setattr__(self, "payload", tuple(payload)) - - def intersect(self, other): - # PackageRestriction defines this but its implementation won't - # work for us, so fail explicitly. - raise NotImplementedError(self) - - def __str__(self): - return "( Conditional: %s payload: [ %s ] )" % ( - PackageRestriction.__str__(self), - ", ".join(map(str, self.payload))) - - def __repr__(self): - if self.negate: - string = '<%s attr=%r restriction=%r payload=%r negated @%#8x>' - else: - string = '<%s attr=%r restriction=%r payload=%r @%#8x>' - return string % ( - self.__class__.__name__, self.attr, self.restriction, self.payload, - id(self)) - - def __iter__(self): - return iter(self.payload) - - def __hash__(self): - return hash((self.attr, self.negate, self.restriction, self.payload)) - - -# "Invalid name" (pylint uses the module const regexp, not the class regexp) -# pylint: disable-msg=C0103 - -AndRestriction = restriction.curry_node_type(boolean.AndRestriction, - restriction.package_type) -OrRestriction = restriction.curry_node_type(boolean.OrRestriction, - restriction.package_type) - -AlwaysBool = restriction.curry_node_type(restriction.AlwaysBool, - restriction.package_type) - -AlwaysTrue = AlwaysBool(negate=True) -AlwaysFalse = AlwaysBool(negate=False) diff --git a/pkgcore/restrictions/restriction.py b/pkgcore/restrictions/restriction.py deleted file mode 100644 index c9f2228..0000000 --- a/pkgcore/restrictions/restriction.py +++ /dev/null @@ -1,200 +0,0 @@ -# Copyright: 2005-2007 Brian Harring <ferringb@gmail.com> -# Copyright: 2006 Marien Zwart <marienz@gentoo.org> -# License: GPL2 - -""" -base restriction class -""" - -from snakeoil import caching -from snakeoil.currying import partial, pretty_docs - -class base(object): - - """ - base restriction matching object. - - all derivatives *should* be __slot__ based (lot of instances may - wind up in memory). - """ - - __metaclass__ = caching.WeakInstMeta - __inst_caching__ = True - - # __weakref__ here's is implicit via the metaclass - __slots__ = () - package_matching = False - - def __setattr__(self, attr, val): - raise TypeError(self, "is immutable") - - def __delattr__(self, attr): - raise TypeError(self, "is immutable") - - def match(self, *arg, **kwargs): - raise NotImplementedError - - def force_False(self, *arg, **kwargs): - return not self.match(*arg, **kwargs) - - def force_True(self, *arg, **kwargs): - return self.match(*arg, **kwargs) - - def intersect(self, other): - return None - - def __len__(self): - return 1 - - -class AlwaysBool(base): - """ - restriction that always yields a specific boolean - """ - __slots__ = ("type", "negate") - - __inst_caching__ = True - - def __init__(self, node_type=None, negate=False): - """ - @param node_type: the restriction type the instance should be, - typically L{pkgcore.restrictions.packages.package_type} or - L{pkgcore.restrictions.values.value_type} - @param negate: boolean to return for the match - """ - object.__setattr__(self, "negate", negate) - object.__setattr__(self, "type", node_type) - - def match(self, *a, **kw): - return self.negate - - def force_True(self, *a, **kw): - return self.negate - - def force_False(self, *a, **kw): - return not self.negate - - def __iter__(self): - return iter(()) - - def __str__(self): - return "always '%s'" % self.negate - - def __repr__(self): - return '<%s always %r @%#8x>' % ( - self.__class__.__name__, self.negate, id(self)) - - -class Negate(base): - - """ - wrap and negate a restriction instance - """ - - __slots__ = ("type", "_restrict") - __inst_caching__ = False - - def __init__(self, restrict): - """ - @param restrict: L{pkgcore.restrictions.restriction.base} instance - to negate - """ - sf = object.__setattr__ - sf(self, "type", restrict.type) - sf(self, "_restrict", restrict) - - def match(self, *a, **kw): - return not self._restrict.match(*a, **kw) - - def __str__(self): - return "not (%s)" % self._restrict - - -class FakeType(base): - - """ - wrapper to wrap and fake a node_type - """ - - __slots__ = ("type", "_restrict") - __inst_caching__ = False - - def __init__(self, restrict, new_type): - """ - @param restrict: L{pkgcore.restrictions.restriction.base} instance - to wrap - @param new_type: new node_type - """ - sf = object.__setattr__ - sf(self, "type", new_type) - sf(self, "_restrict", restrict) - - def match(self, *a, **kw): - return self._restrict.match(*a, **kw) - - def __str__(self): - return "Faked type(%s): %s" % (self.type, self._restrict) - - -class AnyMatch(base): - - """Apply a nested restriction to every item in a sequence.""" - - __slots__ = ('restriction', 'type', 'negate') - - def __init__(self, childrestriction, node_type, negate=False): - """Initialize. - - @type childrestriction: restriction - @param childrestriction: child restriction applied to every value. - @type node_type: string - @param node_type: type of this restriction. - """ - sf = object.__setattr__ - sf(self, "negate", negate) - sf(self, "restriction", childrestriction) - sf(self, "type", node_type) - - def match(self, val): - for x in val: - if self.restriction.match(x): - return not self.negate - return self.negate - - def __str__(self): - return "any: %s match" % (self.restriction,) - - def __repr__(self): - return '<%s restriction=%r @%#8x>' % ( - self.__class__.__name__, self.restriction, id(self)) - - -def curry_node_type(klass, node_type, extradoc=None): - """Helper function for creating restrictions of a certain type. - - This uses L{partial} to pass a node_type to the wrapped class, - and extends the docstring. - - @param klass: callable (usually a class) that is wrapped. - @param node_type: value passed as node_type. - @param extradoc: addition to the docstring. Defaults to - "Automatically set to %s type." % node_type - - @return: a wrapped callable. - """ - if extradoc is None: - extradoc = "Automatically set to %s type." % (node_type,) - doc = klass.__doc__ - result = partial(klass, node_type=node_type) - if doc is None: - doc = '' - else: - # do this so indentation on pydoc __doc__ is sane - doc = "\n".join(line.lstrip() for line in doc.split("\n")) + "\n" - doc += extradoc - return pretty_docs(result, doc) - - -value_type = "values" -package_type = "package" -valid_types = (value_type, package_type) diff --git a/pkgcore/restrictions/util.py b/pkgcore/restrictions/util.py deleted file mode 100644 index e926000..0000000 --- a/pkgcore/restrictions/util.py +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright: 2006 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -""" -restriction related utilities -""" - -from pkgcore.restrictions import packages, boolean, restriction -from snakeoil.lists import iflatten_func - -def _is_package_instance(inst): - return (getattr(inst, "type", None) == packages.package_type - and not isinstance(inst, boolean.base)) - -def collect_package_restrictions(restrict, attrs=None): - """Collect PackageRestriction instances inside a restriction. - - @param restrict: package instance to scan - @param attrs: None (return all package restrictions), or a sequence of - specific attrs the package restriction must work against. - """ - if not isinstance(restrict, (list, tuple)): - restrict = [restrict] - for r in restrict: - if not isinstance(r, restriction.base): - raise TypeError( - "restrict must be of a restriction.base, not %s: %r" % ( - r.__class__.__name__, r)) - if attrs is None: - for r in iflatten_func(restrict, _is_package_instance): - yield r - else: - if isinstance(attrs, (list, tuple)): - attrs = frozenset(attrs) - for r in iflatten_func(restrict, _is_package_instance): - if getattr(r, "attr", None) in attrs: - yield r diff --git a/pkgcore/restrictions/values.py b/pkgcore/restrictions/values.py deleted file mode 100644 index 8e7b7fd..0000000 --- a/pkgcore/restrictions/values.py +++ /dev/null @@ -1,685 +0,0 @@ -# Copyright: 2005-2007 Brian Harring <ferringb@gmail.com> -# Copyright: 2006 Marien Zwart <marienz@gentoo.org> -# License: GPL2 - -""" -value restrictions - -works hand in hand with L{pkgcore.restrictions.packages}, these -classes match against a value handed in, package restrictions pull the -attr from a package instance and hand it to their wrapped restriction -(which is a value restriction). -""" - -from pkgcore.restrictions import restriction, boolean, packages -from snakeoil.klass import generic_equality -from snakeoil import demandload -demandload.demandload(globals(), 're', 'snakeoil:lists') - -# Backwards compatibility. -value_type = restriction.value_type - -try: - from pkgcore.restrictions import _restrictions as extension -except ImportError: - extension = None - -class base(restriction.base): - """Base restriction matching object for values. - - Beware: do not check for instances of this to detect value - restrictions! Use the C{type} attribute instead. - """ - - __slots__ = () - - type = restriction.value_type - - def force_True(self, pkg, attr, val): - return self.match(val) - - def force_False(self, pkg, attr, val): - return not self.match(val) - - -def reflective_hash(self): - return self._hash - -class hashed_base(base): - - __slots__ = ("_hash") - __hash__ = reflective_hash - - -class GetAttrRestriction(packages.PackageRestriction): - - """Restriction pulling an attribute and applying a child restriction.""" - - __slots__ = () - type = restriction.value_type - - # XXX this needs further thought. - # - # The api for force_{True,False} is a ValueRestriction gets called - # with a package instance, the attribute name (string), and the - # current attribute value. We cannot really provide a child - # restriction with a sensible pkg and a sensible attribute name, - # so we just punt and return True/False depending on the current - # state without "forcing" anything (default implementation in - # "base"). - - def force_True(self, pkg, attr, val): - return self.match(val) - - def force_False(self, pkg, attr, val): - return not self.match(val) - - -class VersionRestriction(base): - """use this as base for version restrictions. - - Gives a clue to what the restriction does. - """ - __slots__ = () - - -class StrRegex(hashed_base): - - """ - regex based matching - """ - - __slots__ = ('flags', 'regex', '_matchfunc', 'ismatch', 'negate') - __metaclass__ = generic_equality - __attr_comparison__ = ('_hash',) + __slots__ - __inst_caching__ = True - - def __init__(self, regex, case_sensitive=True, match=False, negate=False): - - """ - @param regex: regex pattern to match - @param case_sensitive: should the match be case sensitive? - @param match: should C{re.match} be used instead of C{re.search}? - @param negate: should the match results be negated? - """ - - sf = object.__setattr__ - sf(self, "regex", regex) - sf(self, "ismatch", match) - sf(self, "negate", negate) - flags = 0 - if not case_sensitive: - flags = re.I - sf(self, "flags", flags) - compiled_re = re.compile(regex, flags) - if match: - sf(self, "_matchfunc", compiled_re.match) - else: - sf(self, "_matchfunc", compiled_re.search) - sf(self, "_hash", hash((self.regex, self.negate, self.flags, self.ismatch))) - - def match(self, value): - if not isinstance(value, basestring): - # Be too clever for our own good --marienz - if value is None: - value = '' - else: - value = str(value) - return (self._matchfunc(value) is not None) != self.negate - - def intersect(self, other): - if self == other: - return self - return None - - def __repr__(self): - result = [self.__class__.__name__, repr(self.regex)] - if self.negate: - result.append('negated') - if self.ismatch: - result.append('match') - else: - result.append('search') - result.append('@%#8x' % (id(self),)) - return '<%s>' % (' '.join(result),) - - def __str__(self): - if self.ismatch: - result = 'match ' - else: - result = 'search ' - result += self.regex - if self.negate: - return 'not ' + result - return result - - -class native_StrExactMatch(object): - - """ - exact string comparison match - """ - - __slots__ = ('_hash', 'exact', 'case_sensitive', 'negate') - __metaclass__ = generic_equality - __attr_comparison__ = __slots__ - - def __init__(self, exact, case_sensitive=True, negate=False): - - """ - @param exact: exact string to match - @param case_sensitive: should the match be case sensitive? - @param negate: should the match results be negated? - """ - - sf = object.__setattr__ - sf(self, "negate", negate) - sf(self, "case_sensitive", case_sensitive) - if not case_sensitive: - sf(self, "exact", str(exact).lower()) - else: - sf(self, "exact", str(exact)) - sf(self, "_hash", hash((self.exact, self.negate, self.case_sensitive))) - - def match(self, value): - if self.case_sensitive: - return (self.exact == value) != self.negate - else: - return (self.exact == value.lower()) != self.negate - - __hash__ = reflective_hash - -if extension is None: - base_StrExactMatch = native_StrExactMatch -else: - base_StrExactMatch = extension.StrExactMatch - -# these are broken out so that it is easier to -# generate native/cpy version of the class for -# testing each. -def _StrExact_intersect(self, other): - s1, s2 = self.exact, other.exact - if other.case_sensitive and not self.case_sensitive: - s1 = s1.lower() - elif self.case_sensitive and not other.case_sensitive: - s2 = s2.lower() - if s1 == s2 and self.negate == other.negate: - if other.case_sensitive: - return other - return self - return None - -def _StrExact__repr__(self): - if self.negate: - string = '<%s %r negated @%#8x>' - else: - string = '<%s %r @%#8x>' - return string % (self.__class__.__name__, self.exact, id(self)) - -def _StrExact__str__(self): - if self.negate: - return "!= "+self.exact - return "== "+self.exact - -class StrExactMatch(base_StrExactMatch, base): - - __slots__ = () - __inst_caching__ = True - - intersect = _StrExact_intersect - __repr__ = _StrExact__repr__ - __str__ = _StrExact__str__ - - -class StrGlobMatch(hashed_base): - - """ - globbing matches; essentially startswith and endswith matches - """ - - __slots__ = ('glob', 'prefix', 'negate', 'flags') - __attr_comparison__ = ('_hash',) + __slots__ - __metaclass__ = generic_equality - __inst_caching__ = True - - def __init__(self, glob, case_sensitive=True, prefix=True, negate=False): - - """ - @param glob: string chunk that must be matched - @param case_sensitive: should the match be case sensitive? - @param prefix: should the glob be a prefix check for matching, - or postfix matching - @param negate: should the match results be negated? - """ - - sf = object.__setattr__ - sf(self, "negate", negate) - if not case_sensitive: - sf(self, "flags", re.I) - sf(self, "glob", str(glob).lower()) - else: - sf(self, "flags", 0) - sf(self, "glob", str(glob)) - sf(self, "prefix", prefix) - sf(self, "_hash", hash((self.glob, self.negate, self.flags, self.prefix))) - - def match(self, value): - value = str(value) - if self.flags == re.I: - value = value.lower() - if self.prefix: - f = value.startswith - else: - f = value.endswith - return f(self.glob) ^ self.negate - - def intersect(self, other): - if self.match(other.glob): - if self.negate == other.negate: - return other - elif other.match(self.glob): - if self.negate == other.negate: - return self - return None - - def __repr__(self): - if self.negate: - string = '<%s %r case_sensitive=%r negated @%#8x>' - else: - string = '<%s %r case_sensitive=%r @%#8x>' - if self.prefix: - g = self.glob + ".*" - else: - g = ".*" + self.glob - return string % (self.__class__.__name__, g, - self.flags == re.I and True or False, - id(self)) - - def __str__(self): - s = '' - if self.negate: - s = 'not ' - if self.prefix: - return "%s%s*" % (s, self.glob) - return "%s*%s" % (s, self.glob) - - -def EqualityMatch(val, negate=False): - """ - equality test wrapping L{ComparisonMatch} - """ - return ComparisonMatch(cmp, val, [0], negate=negate) - -def _mangle_cmp_val(val): - if val < 0: - return -1 - elif val > 0: - return 1 - return 0 - - -class ComparisonMatch(hashed_base): - """Match if the comparison funcs return value is what's required.""" - - _op_converter = {"=": (0,)} - _rev_op_converter = {(0,): "="} - - for k, v in (("<", (-1,)), (">", (1,))): - _op_converter[k] = v - _op_converter[k+"="] = tuple(sorted(v + (0,))) - _rev_op_converter[v] = k - _rev_op_converter[tuple(sorted(v+(0,)))] = k+"=" - _op_converter["!="] = _op_converter["<>"] = (-1, 1) - _rev_op_converter[(-1, 1)] = "!=" - del k, v - - __slots__ = ('cmp_func', 'data', 'matching_vals') - __metaclass__ = generic_equality - __attr_comparison__ = __slots__ - - @classmethod - def convert_str_op(cls, op_str): - return cls._op_converter[op_str] - - @classmethod - def convert_op_str(cls, op): - return cls._rev_op_converter[tuple(sorted(op))] - - def __init__(self, cmp_func, data, matching_vals, negate=False): - - """ - @param cmp_func: comparison function that compares data against what - is passed in during match - @param data: data to base comparison against - @param matching_vals: sequence, composed of - [-1 (less then), 0 (equal), and 1 (greater then)]. - If you specify [-1,0], you're saying - "result must be less then or equal to". - @param negate: should the results be negated? - """ - - sf = object.__setattr__ - sf(self, "cmp_func", cmp_func) - - if not isinstance(matching_vals, (tuple, list)): - if isinstance(matching_vals, basestring): - matching_vals = self.convert_str_op(matching_vals) - elif isinstance(matching_vals, int): - matching_vals = [matching_vals] - else: - raise TypeError("matching_vals must be a list/tuple") - - sf(self, "data", data) - if negate: - sf(self, "matching_vals", - tuple(set([-1, 0, 1]).difference(_mangle_cmp_val(x) - for x in matching_vals))) - else: - sf(self, "matching_vals", - tuple(_mangle_cmp_val(x) for x in matching_vals)) - - def __hash__(self): - return hash((self.cmp_func, self.matching_vals, self.data)) - - def match(self, actual_val): - return _mangle_cmp_val( - self.cmp_func(actual_val, self.data)) in self.matching_vals - - def __repr__(self): - return '<%s %s %r @%#8x>' % ( - self.__class__.__name__, self.convert_op_str(self.matching_vals), - self.data, id(self)) - - def __str__(self): - return "%s %s" % (self.convert_op_str(self.matching_vals), self.data) - - -class ContainmentMatch(hashed_base): - - """used for an 'in' style operation, 'x86' in ['x86','~x86'] for example - note that negation of this *does* not result in a true NAND when all is on. - """ - - __slots__ = ('vals', 'all', 'negate') - __metaclass__ = generic_equality - __attr_comparison__ = ('_hash',) + __slots__ - __inst_caching__ = True - - def __init__(self, *vals, **kwds): - - """ - @param vals: what values to look for during match - @keyword all: must all vals be present, or just one for a match - to succeed? - @keyword negate: should the match results be negated? - """ - - sf = object.__setattr__ - sf(self, "all", bool(kwds.pop("all", False))) - - # note that we're discarding any specialized __getitem__ on vals here. - # this isn't optimal, and should be special cased for known - # types (lists/tuples fex) - sf(self, "vals", frozenset(vals)) - sf(self, "negate", kwds.get("negate", False)) - sf(self, "_hash", hash((self.all, self.negate, self.vals))) - - def match(self, val): - if isinstance(val, basestring): - for fval in self.vals: - if fval in val: - return not self.negate - return self.negate - - # this can, and should be optimized to do len checks- iterate - # over the smaller of the two see above about special casing - # bits. need the same protection here, on the offchance (as - # contents sets do), the __getitem__ is non standard. - try: - if self.all: - i = iter(val) - return bool(self.vals.difference(i)) == self.negate - for x in self.vals: - if x in val: - return not self.negate - return self.negate - except TypeError: - # other way around. rely on contains. - if self.all: - for k in self.vals: - if k not in val: - return self.negate - return not self.negate - for k in self.vals: - if k in val: - return not self.negate - - - def force_False(self, pkg, attr, val): - - # "More than one statement on a single line" - # pylint: disable-msg=C0321 - - # XXX pretty much positive this isn't working. - if isinstance(val, basestring) or not getattr(pkg, 'configurable', - False): - # unchangable - return not self.match(val) - - if self.negate: - if self.all: - def filter(truths): - return False in truths - def true(r, pvals): - return pkg.request_enable(attr, r) - def false(r, pvals): - return pkg.request_disable(attr, r) - - truths = [x in val for x in self.vals] - - for x in boolean.iterative_quad_toggling( - pkg, None, list(self.vals), 0, len(self.vals), truths, - filter, desired_false=false, desired_true=true): - return True - else: - if pkg.request_disable(attr, *self.vals): - return True - return False - - if not self.all: - if pkg.request_disable(attr, *self.vals): - return True - else: - l = len(self.vals) - def filter(truths): return truths.count(True) < l - def true(r, pvals): return pkg.request_enable(attr, r) - def false(r, pvals): return pkg.request_disable(attr, r) - truths = [x in val for x in self.vals] - for x in boolean.iterative_quad_toggling( - pkg, None, list(self.vals), 0, l, truths, filter, - desired_false=false, desired_true=true): - return True - return False - - - def force_True(self, pkg, attr, val): - - # "More than one statement on a single line" - # pylint: disable-msg=C0321 - - # XXX pretty much positive this isn't working. - - if isinstance(val, basestring) or not getattr(pkg, 'configurable', - False): - # unchangable - return self.match(val) - - if not self.negate: - if not self.all: - def filter(truths): - return True in truths - def true(r, pvals): - return pkg.request_enable(attr, r) - def false(r, pvals): - return pkg.request_disable(attr, r) - - truths = [x in val for x in self.vals] - - for x in boolean.iterative_quad_toggling( - pkg, None, list(self.vals), 0, len(self.vals), truths, - filter, desired_false=false, desired_true=true): - return True - else: - if pkg.request_enable(attr, *self.vals): - return True - return False - - # negation - if not self.all: - if pkg.request_disable(attr, *self.vals): - return True - else: - def filter(truths): return True not in truths - def true(r, pvals): return pkg.request_enable(attr, r) - def false(r, pvals): return pkg.request_disable(attr, r) - truths = [x in val for x in self.vals] - for x in boolean.iterative_quad_toggling( - pkg, None, list(self.vals), 0, len(self.vals), truths, filter, - desired_false=false, desired_true=true): - return True - return False - - def __repr__(self): - if self.negate: - string = '<%s %r all=%s negated @%#8x>' - else: - string = '<%s %r all=%s @%#8x>' - return string % ( - self.__class__.__name__, tuple(self.vals), self.all, id(self)) - - def __str__(self): - if self.negate: - s = "not contains [%s]" - else: - s = "contains [%s]" - return s % ', '.join(map(str, self.vals)) - - -class FlatteningRestriction(hashed_base): - - """Flatten the values passed in and apply the nested restriction.""" - - __slots__ = ('dont_iter', 'restriction', 'negate') - __hash__ = object.__hash__ - - def __init__(self, dont_iter, childrestriction, negate=False): - """Initialize. - - @type dont_iter: type or tuple of types - @param dont_iter: type(s) not to flatten. - Passed to L{snakeoil.lists.iflatten_instance}. - @type childrestriction: restriction - @param childrestriction: restriction applied to the flattened list. - """ - object.__setattr__(self, "negate", negate) - object.__setattr__(self, "dont_iter", dont_iter) - object.__setattr__(self, "restriction", childrestriction) - - def match(self, val): - return self.restriction.match( - lists.iflatten_instance(val, self.dont_iter)) != self.negate - - def __str__(self): - return 'flattening_restriction: dont_iter = %s, restriction = %s' % ( - self.dont_iter, self.restriction) - - def __repr__(self): - return '<%s restriction=%r dont_iter=%r negate=%r @%#8x>' % ( - self.__class__.__name__, - self.restriction, self.dont_iter, self.negate, - id(self)) - - -class FunctionRestriction(hashed_base): - - """Convenience class for creating special restrictions.""" - - __slots__ = ('func', 'negate') - - __hash__ = object.__hash__ - - def __init__(self, func, negate=False): - """Initialize. - - C{func} is used as match function. - - It will usually be impossible for the backend to optimize this - restriction. So even though you can implement an arbitrary - restriction using this class you should only use it if it is - very unlikely backend-specific optimizations will be possible. - """ - object.__setattr__(self, 'negate', negate) - object.__setattr__(self, 'func', func) - - def match(self, val): - return self.func(val) != self.negate - - def __repr__(self): - return '<%s func=%r negate=%r @%#8x>' % ( - self.__class__.__name__, self.func, self.negate, id(self)) - - -class StrConversion(hashed_base): - - """convert passed in data to a str object""" - - __hash__ = object.__hash__ - __slots__ = ('restrict',) - - def __init__(self, restrict): - object.__setattr__(self, "restrict", restrict) - - def match(self, val): - return self.restrict.match(str(val)) - - -class UnicodeConversion(StrConversion): - - """convert passed in data to a unicode obj""" - - def match(self, val): - return self.restrict.match(unicode(val)) - - -class AnyMatch(restriction.AnyMatch): - - __slots__ = () - - __hash__ = object.__hash__ - - def __init__(self, childrestriction, negate=False): - # Hack: skip calling base.__init__. Doing this would make - # restriction.base.__init__ run twice. - restriction.AnyMatch.__init__( - self, childrestriction, restriction.value_type, negate=negate) - - def force_True(self, pkg, attr, val): - return self.match(val) - - def force_False(self, pkg, attr, val): - return not self.match(val) - - -# "Invalid name" (pylint uses the module const regexp, not the class regexp) -# pylint: disable-msg=C0103 - -AndRestriction = restriction.curry_node_type(boolean.AndRestriction, - restriction.value_type) -OrRestriction = restriction.curry_node_type(boolean.OrRestriction, - restriction.value_type) - -AlwaysBool = restriction.curry_node_type(restriction.AlwaysBool, - restriction.value_type) - -AlwaysTrue = AlwaysBool(negate=True) -AlwaysFalse = AlwaysBool(negate=False) diff --git a/pkgcore/scripts/__init__.py b/pkgcore/scripts/__init__.py deleted file mode 100644 index 089357c..0000000 --- a/pkgcore/scripts/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright: 2006 Marien Zwart <marienz@gentoo.org> -# License: GPL2 - -"""Commandline scripts. - -Modules in here are accessible through the pwrapper script. They -should have an C{OptionParser} attribute that is a -L{snakeoil.commandline.OptionParser} subclass and a C{main} -attribute that is a function usable with -L{snakeoil.commandline.main}. - -The goal of this is avoiding boilerplate and making sure the scripts -have a similar look and feel. If your script needs to do something -L{snakeoil.commandline} does not support please improve it instead -of bypassing it. -""" diff --git a/pkgcore/scripts/filter_env.py b/pkgcore/scripts/filter_env.py deleted file mode 100644 index d83b333..0000000 --- a/pkgcore/scripts/filter_env.py +++ /dev/null @@ -1,95 +0,0 @@ -# Copyright: 2006 Marien Zwart <marienz@gentoo.org> -# License: GPL2 - -"""Commandline interface to L{pkgcore.ebuild.filter_env}.""" - - -import sys - -from pkgcore.util import commandline -# ordering here matters; commandline does a trick to copy to avoid the heavy inspect load. -import optparse -from pkgcore.ebuild import filter_env -from pkgcore.log import logger - - -def input_callback(option, opt_str, value, parser): - if parser.values.input is not None: - raise optparse.OptionValueError('-i cannot be specified twice') - try: - parser.values.input = open(value, 'r') - except (IOError, OSError), e: - raise optparse.OptionValueError('error opening %r (%s)' % (value, e)) - - -def append_comma_separated(option, opt_str, value, parser): - parser.values.ensure_value(option.dest, []).extend( - v for v in value.split(',') if v) - - -class OptionParser(commandline.OptionParser): - - def __init__(self, **kwargs): - commandline.OptionParser.__init__(self, **kwargs) - self.add_option( - '-V', '--var-match', action='store_false', default=True) - self.add_option( - '-F', '--func-match', action='store_false', default=True) - self.add_option( - '--input', '-i', action='callback', type='string', - callback=input_callback, - help='Filename to read the env from (uses stdin if omitted).') - self.add_option( - '--funcs', '-f', action='callback', type='string', - callback=append_comma_separated) - self.add_option( - '--vars', '-v', action='callback', type='string', - callback=append_comma_separated) - - def check_values(self, values, args): - values, args = commandline.OptionParser.check_values( - self, values, args) - - if values.input is None: - # Hack: use stdin if it is not a tty. No util.commandline - # support for this kind of thing, so mess around with sys - # directly. - if sys.stdin.isatty(): - self.error('No input file supplied (and stdin is a tty).') - values.input = sys.stdin - - return values, args - - -def main(options, out, err): - if options.debug: - if options.funcs is None: - logger.debug('=== Funcs: None') - else: - logger.debug('=== Funcs:') - for thing in options.funcs: - logger.debug(repr(thing)) - if options.vars is None: - logger.debug('=== Vars: None') - else: - logger.debug('=== Vars:') - for thing in options.vars: - logger.debug(repr(thing)) - logger.debug('var_match: %r, func_match: %r', - options.var_match, options.func_match) - - if options.funcs: - funcs = filter_env.build_regex_string(options.funcs) - else: - funcs = None - - if options.vars: - vars = filter_env.build_regex_string(options.vars) - else: - vars = None - - file_buff = options.input.read() + '\0' - - # Hack: write to the stream directly. - filter_env.run(out.stream, file_buff, vars, funcs, - options.var_match, options.func_match) diff --git a/pkgcore/scripts/pclone_cache.py b/pkgcore/scripts/pclone_cache.py deleted file mode 100644 index b55269e..0000000 --- a/pkgcore/scripts/pclone_cache.py +++ /dev/null @@ -1,73 +0,0 @@ -# Copyright: 2005-2006 Brian Harring <ferringb@gmail.com> -# Copyright: 2006 Marien Zwart <marienz@gentoo.org> -# License: GPL2 - - -"""Clone a repository cache.""" - - -import time - -from pkgcore.util import commandline - - -class OptionParser(commandline.OptionParser): - - def __init__(self, **kwargs): - commandline.OptionParser.__init__( - self, description=__doc__, usage='%prog [options] source target', - **kwargs) - self.add_option('--verbose', '-v', action='store_true', - help='print keys as they are processed') - - def check_values(self, values, args): - values, args = commandline.OptionParser.check_values( - self, values, args) - if len(args) != 2: - self.error( - 'Need two arguments: cache label to read from and ' - 'cache label to write to.') - - config = values.config - try: - values.source = config.cache[args[0]] - except KeyError: - self.error("read cache label '%s' isn't defined." % (args[0],)) - try: - values.target = config.cache[args[1]] - except KeyError: - self.error("write cache label '%s' isn't defined." % (args[1],)) - - if values.target.readonly: - self.error("can't update cache label '%s', it's marked readonly." % - (args[1],)) - - return values, () - - -def main(options, out, err): - source, target = options.source, options.target - if not target.autocommits: - target.sync_rate = 1000 - if options.verbose: - out.write("grabbing target's existing keys") - valid = set() - start = time.time() - if options.verbose: - for k, v in source.iteritems(): - out.write("updating %s" % (k,)) - target[k] = v - valid.add(k) - else: - for k, v in source.iteritems(): - target[k] = v - valid.add(k) - - for x in target.iterkeys(): - if not x in valid: - if options.verbose: - out.write("deleting %s" % (x,)) - del target[x] - - if options.verbose: - out.write("took %i seconds" % int(time.time() - start)) diff --git a/pkgcore/scripts/pconfig.py b/pkgcore/scripts/pconfig.py deleted file mode 100644 index 06c06f5..0000000 --- a/pkgcore/scripts/pconfig.py +++ /dev/null @@ -1,317 +0,0 @@ -# Copyright: 2006 Marien Zwart <marienz@gentoo.org> -# License: GPL2 - -"""Configuration querying utility.""" - - -import traceback - -from pkgcore.config import errors, basics -from pkgcore.plugin import get_plugins -from pkgcore.util import commandline -from snakeoil import modules - -class DescribeClassParser(commandline.OptionParser): - - """Our option parser.""" - - def check_values(self, values, args): - values, args = commandline.OptionParser.check_values( - self, values, args) - if len(args) != 1: - self.error('need exactly one argument: class to describe.') - try: - values.describe_class = modules.load_attribute(args[0]) - except modules.FailedImport, e: - self.error(str(e)) - return values, () - - -def dump_section(config, out): - out.first_prefix.append(' ') - out.write('# typename of this section: %s' % (config.type.name,)) - out.write('class %s.%s;' % (config.type.callable.__module__, - config.type.callable.__name__)) - if config.default: - out.write('default true;') - for key, val in sorted(config.config.iteritems()): - typename = config.type.types.get(key) - if typename is None: - if config.type.allow_unknowns: - typename = 'str' - else: - raise ValueError('no type set for %s (%r)' % (key, val)) - out.write('# type: %s' % (typename,)) - if typename.startswith('lazy_refs'): - typename = typename[5:] - val = list(ref.collapse() for ref in val) - elif typename.startswith('lazy_ref'): - typename = typename[5:] - val = val.collapse() - if typename == 'str': - out.write('%s %r;' % (key, val)) - elif typename == 'bool': - out.write('%s %s;' % (key, bool(val))) - elif typename == 'list': - out.write('%s %s;' % ( - key, ' '.join(repr(string) for string in val))) - elif typename == 'callable': - out.write('%s %s.%s;' % (key, val.__module__, val.__name__)) - elif typename.startswith('ref:'): - if val.name is None: - out.write('%s {' % (key,)) - dump_section(val, out) - out.write('};') - else: - out.write('%s %r;' % (key, val.name)) - elif typename.startswith('refs:'): - out.autoline = False - out.write('%s' % (key,)) - for i, subconf in enumerate(val): - if subconf.name is None: - out.autoline = True - out.write(' {') - dump_section(subconf, out) - out.autoline = False - out.write('}') - else: - out.write(' %r' % (subconf.name,)) - out.autoline = True - out.write(';') - else: - out.write('# %s = %r of unknown type %s' % (key, val, typename)) - out.first_prefix.pop() - - -def get_classes(configs): - # Not particularly efficient (doesn't memoize already visited configs) - classes = set() - for config in configs: - classes.add('%s.%s' % (config.type.callable.__module__, - config.type.callable.__name__)) - for key, val in config.config.iteritems(): - typename = config.type.types.get(key) - if typename is None: - continue - if typename.startswith('ref:'): - classes.update(get_classes((val,))) - elif typename.startswith('refs:'): - classes.update(get_classes(val)) - elif typename.startswith('lazy_refs'): - classes.update(get_classes(c.collapse() for c in val)) - elif typename.startswith('lazy_ref'): - classes.update(get_classes((val.collapse(),))) - return classes - -def classes_main(options, out, err): - """List all classes referenced by the config.""" - configmanager = options.config - sections = [] - for name in configmanager.sections(): - try: - sections.append(configmanager.collapse_named_section(name)) - except errors.CollapseInheritOnly: - pass - for classname in sorted(get_classes(sections)): - out.write(classname) - - -def write_type(out, type_obj): - out.write('typename is %s' % (type_obj.name,)) - if type_obj.doc: - for line in type_obj.doc.split('\n'): - out.write(line.strip(), wrap=True) - if type_obj.allow_unknowns: - out.write('values not listed are handled as strings') - out.write() - for name, typename in sorted(type_obj.types.iteritems()): - out.write('%s: %s' % (name, typename), autoline=False) - if name in type_obj.required: - out.write(' (required)', autoline=False) - out.write() - - -def describe_class_main(options, out, err): - """Describe the arguments a class needs.""" - try: - type_obj = basics.ConfigType(options.describe_class) - except errors.TypeDefinitionError: - err.write('Not a valid type!') - return 1 - write_type(out, type_obj) - - -def uncollapsable_main(options, out, err): - """Show things that could not be collapsed.""" - config = options.config - for name in config.sections(): - try: - config.collapse_named_section(name) - except errors.CollapseInheritOnly: - pass - except errors.ConfigurationError, e: - if options.debug: - traceback.print_exc() - else: - out.write(str(e)) - out.write() - - -class _TypeNameParser(commandline.OptionParser): - - """Base for subcommands that take an optional type name.""" - - def check_values(self, values, args): - values, args = commandline.OptionParser.check_values(self, values, - args) - if len(args) > 1: - self.error('pass at most one typename') - if args: - values.typename = args[0] - else: - values.typename = None - return values, () - - -class DumpParser(_TypeNameParser): - - def __init__(self, **kwargs): - # Make sure we do not pass two description kwargs if kwargs has one. - kwargs['description'] = ( - 'Dump the entire configuration. ' - 'The format used is similar to the ini-like default ' - 'format, but do not rely on this to always write a ' - 'loadable config. There may be quoting issues. ' - 'With a typename argument only that type is dumped.') - kwargs['usage'] = '%prog [options] [typename]' - _TypeNameParser.__init__(self, **kwargs) - - -def dump_main(options, out, err): - """Dump the entire configuration.""" - config = options.config - if options.typename is None: - names = config.sections() - else: - names = getattr(config, options.typename).iterkeys() - for name in sorted(names): - try: - section = config.collapse_named_section(name) - except errors.CollapseInheritOnly: - continue - out.write('%r {' % (name,)) - dump_section(section, out) - out.write('}') - out.write() - - -class ConfigurablesParser(_TypeNameParser): - - def __init__(self, **kwargs): - # Make sure we do not pass two description kwargs if kwargs has one. - kwargs['description'] = ( - 'List registered configurables (may not be complete). ' - 'With a typename argument only configurables of that type are ' - 'listed.') - kwargs['usage'] = '%prog [options] [typename]' - _TypeNameParser.__init__(self, **kwargs) - - -def configurables_main(options, out, err): - """List registered configurables.""" - for configurable in get_plugins('configurable'): - type_obj = basics.ConfigType(configurable) - if options.typename is not None and type_obj.name != options.typename: - continue - out.write(out.bold, '%s.%s' % ( - configurable.__module__, configurable.__name__)) - write_type(out, type_obj) - out.write() - out.write() - - -def _dump_uncollapsed_section(config, out, err, section): - """Write a single section.""" - if isinstance(section, basestring): - out.write('named section %r' % (section,)) - return - for key in sorted(section.keys()): - kind, value = section.get_value(config, key, 'repr') - out.write('# type: %s' % (kind,)) - if kind == 'list': - for name, val in zip(( - key + '.prepend', key, key + '.append'), value): - if val: - out.write( - repr(name), ' = ', ' '.join(repr(v) for v in val)) - continue - if kind in ('refs', 'str'): - for name, val in zip(( - key + '.prepend', key, key + '.append'), value): - if not val: - continue - out.write(repr(name), ' = ', autoline=False) - if kind == 'str': - out.write(repr(val)) - else: - out.write() - out.first_prefix.append(' ') - try: - for subnr, subsection in enumerate(val): - subname = 'nested section %s' % (subnr + 1,) - out.write(subname) - out.write('=' * len(subname)) - _dump_uncollapsed_section(config, out, err, subsection) - out.write() - finally: - out.first_prefix.pop() - continue - out.write('%r = ' % (key,), autoline=False) - if kind == 'callable': - out.write(value.__module__, value.__name__) - elif kind == 'bool': - out.write(str(value)) - elif kind == 'ref': - out.first_prefix.append(' ') - try: - out.write() - _dump_uncollapsed_section(config, out, err, value) - finally: - out.first_prefix.pop() - else: - err.error('unsupported type %r' % (kind,)) - - -def dump_uncollapsed_main(options, out, err): - """dump the configuration in a raw, uncollapsed form. - Not directly usable as a configuration file, mainly used for inspection - """ - out.write('''# Warning: -# Do not copy this output to a configuration file directly, -# because the types you see here are only guesses. -# A value used as "list" in the collapsed config will often -# show up as "string" here and may need to be converted -# (for example from space-separated to comma-separated) -# to work in a config file with a different format. -''') - for i, source in enumerate(options.config.configs): - s = 'Source %s' % (i + 1,) - out.write(out.bold, '*' * len(s)) - out.write(out.bold, s) - out.write(out.bold, '*' * len(s)) - out.write() - for name, section in sorted(source.iteritems()): - out.write('%s' % (name,)) - out.write('=' * len(name)) - _dump_uncollapsed_section(options.config, out, err, section) - out.write() - - -commandline_commands = { - 'dump': (DumpParser, dump_main), - 'classes': (commandline.OptionParser, classes_main), - 'uncollapsable': (commandline.OptionParser, uncollapsable_main), - 'describe_class': (DescribeClassParser, describe_class_main), - 'configurables': (ConfigurablesParser, configurables_main), - 'dump-uncollapsed': (commandline.OptionParser, dump_uncollapsed_main), - } diff --git a/pkgcore/scripts/pebuild.py b/pkgcore/scripts/pebuild.py deleted file mode 100644 index 4f40078..0000000 --- a/pkgcore/scripts/pebuild.py +++ /dev/null @@ -1,57 +0,0 @@ -# Copyright: 2005-2007 Brian Harring <ferringb@gmail.com> -# Copyright: 2006 Marien Zwart <marienz@gentoo.org> -# License: GPL2 - - -"""Low-level ebuild operations.""" - - -from pkgcore.util import commandline -from pkgcore.ebuild import atom, errors - - -class OptionParser(commandline.OptionParser): - - def __init__(self, **kwargs): - commandline.OptionParser.__init__( - self, description=__doc__, usage='%prog [options] atom phases', - **kwargs) - self.add_option("--no-auto", action='store_true', default=False, - help="run just the specified phases. may explode.") - - def check_values(self, values, args): - values, args = commandline.OptionParser.check_values( - self, values, args) - if len(args) < 2: - self.error('Specify an atom and at least one phase.') - try: - values.atom = atom.atom(args[0]) - except errors.MalformedAtom, e: - self.error(str(e)) - values.phases = args[1:] - return values, () - -def main(options, out, err): - pkgs = options.config.get_default('domain').all_repos.match(options.atom) - if not pkgs: - err.write('got no matches for %s\n' % (options.atom,)) - return 1 - if len(pkgs) > 1: - err.write('got multiple matches for %s: %s\n' % (options.atom, pkgs)) - return 1 - # pull clean out. - l = list(x for x in options.phases if x != "clean") - clean = len(l) != len(options.phases) - if clean: - options.phases = l - kwds = {} - if options.no_auto: - kwds["ignore_deps"] = True - if "setup" in l: - options.phases.insert(0, "fetch") - build = pkgs[0].build(clean=clean) - phase_funcs = list(getattr(build, x) for x in options.phases) - for phase, f in zip(options.phases, phase_funcs): - out.write() - out.write('executing phase %s' % (phase,)) - f(**kwds) diff --git a/pkgcore/scripts/pmaint.py b/pkgcore/scripts/pmaint.py deleted file mode 100644 index d61bc43..0000000 --- a/pkgcore/scripts/pmaint.py +++ /dev/null @@ -1,393 +0,0 @@ -# Copyright: 2005-2007 Brian Harring <ferringb@gmail.com> -# Copyright: 2006 Marien Zwart <marienz@gentoo.org> -# License: GPL2 - -""" -repository maintainence -""" - -__all__ = ('CopyParser', 'DigestParser', 'RegenParser', 'SyncParser') - -from pkgcore.util.commandline import convert_to_restrict, OptionParser -from snakeoil.demandload import demandload -demandload(globals(), - 'errno', - 'threading:Event', - 'threading:Thread', - 'Queue:Queue,Empty', - 'time:time,sleep', - 'snakeoil.osutils:pjoin', - 'pkgcore.repository:multiplex', - 'pkgcore.package:mutated', - 'pkgcore.fs:contents,livefs', - 'pkgcore.ebuild:atom,errors,digest', - 'pkgcore.restrictions.boolean:OrRestriction', -) - -commandline_commands = {} - -def format_seq(seq, formatter=repr): - if not seq: - seq = None - elif len(seq) == 1: - seq = seq[0] - else: - seq = tuple(sorted(str(x) for x in seq)) - return formatter(seq) - - -class SyncParser(OptionParser): - - def __init__(self, **kwargs): - OptionParser.__init__(self, description= - "update a local repository to match its parent", **kwargs) - self.add_option("--force", action='store_true', default=False, - help="force an action") - - def check_values(self, values, args): - values, args = OptionParser.check_values( - self, values, args) - - if not args: - values.repos = values.config.repo.keys() - else: - for x in args: - if x not in values.config.repo: - self.error("repo %r doesn't exist:\nvalid repos %r" % - (x, values.config.repo.keys())) - values.repos = args - return values, [] - -def sync_main(options, out, err): - """Update a local repositories to match their remote parent""" - config = options.config - succeeded, failed = [], [] - seen = set() - for x in options.repos: - r = config.repo[x] - if r in seen: - continue - seen.add(r) - if not r.syncable: - continue - out.write("*** syncing %r..." % x) - if not r.sync(force=options.force): - out.write("*** failed syncing %r" % x) - failed.append(x) - else: - succeeded.append(x) - out.write("*** synced %r" % x) - if len(succeeded) + len(failed) > 1: - out.write("*** synced %s" % format_seq(sorted(succeeded))) - if failed: - err.write("!!! failed sync'ing %s" % format_seq(sorted(failed))) - if failed: - return 1 - return 0 - -commandline_commands['sync'] = (SyncParser, sync_main) - - -class CopyParser(OptionParser): - - def __init__(self, **kwargs): - OptionParser.__init__(self, description= - "copy built pkg(s) into a repository", **kwargs) - self.add_option("-s", "--source-repo", - help="copy from just the specified repository; else defaults " - "to finding any match") - self.add_option("-t", "--target-repo", default=None, - help="repository to copy packages into; if specified, " - "you don't need to specify the target repo as the last arg. " - "Mainly useful for xargs invocations") - self.add_option("--ignore-existing", "-i", default=False, - action='store_true', - help="skip existing pkgs, instead of treating it as an overwrite " - "error") - self.add_option("--copy-missing", action="store_true", default=False, - help="Copy packages missing in target repo from source repo") - self.add_option("--force", action='store_true', default=False, - help="try and force the copy if the target repository is marked as " - "immutable") - - def check_values(self, values, args): - l = len(args) - if not values.target_repo and l < 2: - self.error("target_report wasn't specified- specify it either as " - "the last arguement, or via --target-repo") - - if values.target_repo is not None: - target_repo = values.target_repo - else: - target_repo = args.pop(-1) - - try: - values.target_repo = values.config.repo[target_repo] - except KeyError: - self.error("target repo %r was not found, known repos-\n%s" % - (target_repo, format_seq(values.config.repo.keys()))) - - if values.target_repo.frozen and not values.force: - self.error("target repo %r is frozen; --force is required to " - "override this" % target_repo) - - if values.source_repo: - try: - values.source_repo = values.config.repo[values.source_repo] - except KeyError: - self.error("source repo %r was not found, known repos-\n%s" % - (values.source_repo, format_seq(values.config.repo.keys()))) - else: - values.source_repo = multiplex.tree(*values.config.repos.values()) - - values.candidates = [] - if values.copy_missing: - restrict = OrRestriction(*convert_to_restrict(args)) - for package in values.source_repo.itermatch(restrict): - if not values.target_repo.match(package.versioned_atom): - values.candidates.append(package.versioned_atom) - else: - values.candidates = convert_to_restrict(args) - - return values, [] - - -def copy_main(options, out, err): - """Copy pkgs between repositories.""" - - trg_repo = options.target_repo - src_repo = options.source_repo - - failures = False - kwds = {'force': options.force} - - for candidate in options.candidates: - matches = src_repo.match(candidate) - if not matches: - err.write("didn't find any matching pkgs for %r" % candidate) - failures = True - continue - - for src in matches: - existing = trg_repo.match(src.versioned_atom) - args = [] - pkg = src - if len(existing) > 1: - err.write( - "skipping %r; tried to replace more then one pkg %r..." % - (src, format_seq(existing))) - failures = True - continue - elif len(existing) == 1: - if options.ignore_existing: - out.write("skipping %s, since %s exists already" % - (src, existing[0])) - continue - out.write("replacing %s with %s... " % (src, existing[0])) - op = trg_repo.replace - args = existing - else: - out.write("copying %s... " % src) - op = trg_repo.install - - if src.repo.livefs: - out.write("forcing regen of contents due to src being livefs..") - new_contents = contents.contentsSet(mutable=True) - for fsobj in src.contents: - try: - new_contents.add(livefs.gen_obj(fsobj.location)) - except OSError, oe: - if oe.errno != errno.ENOENT: - err.write("failed accessing fs obj %r; %r\n" - "aborting this copy" % - (fsobj, oe)) - failures = True - new_contents = None - break - err.write("warning: dropping fs obj %r since it " - "doesn't exist" % fsobj) - if new_contents is None: - continue - pkg = mutated.MutatedPkg(src, {'contents':new_contents}) - - op = op(*(args + [pkg]), **kwds) - op.finish() - - out.write("completed\n") - if failures: - return 1 - return 0 - -commandline_commands['copy'] = (CopyParser, copy_main) - - -class RegenParser(OptionParser): - - def __init__(self, **kwargs): - OptionParser.__init__( - self, description=__doc__, usage='%prog [options] repo [threads]', - **kwargs) - - def check_values(self, values, args): - values, args = OptionParser.check_values( - self, values, args) - if not args: - self.error('Need a repository name.') - if len(args) > 2: - self.error('I do not know what to do with more than 2 arguments') - - if len(args) == 2: - try: - values.thread_count = int(args[1]) - except ValueError: - self.error('%r should be an integer' % (args[1],)) - if values.thread_count <= 0: - self.error('thread count needs to be at least 1') - else: - values.thread_count = 1 - - try: - values.repo = values.config.repo[args[0]] - except KeyError: - self.error('repo %r was not found! known repos: %s' % ( - args[0], ', '.join(str(x) for x in values.config.repo))) - - return values, () - - -def regen_iter(iterable, err): - for x in iterable: - try: - x.keywords - except RuntimeError: - raise - except Exception, e: - err.write("caught exception %s for %s" % (e, x)) - -def reclaim_threads(threads, err): - for x in threads: - try: - x.join() - except RuntimeError: - raise - except Exception, e: - err.write("caught exception %s reclaiming thread" % (e,)) - -def regen_main(options, out, err): - """Regenerate a repository cache.""" - start_time = time() - # HACK: store this here so we can assign to it from inside def passthru. - options.count = 0 - if options.thread_count == 1: - def passthru(iterable): - for x in iterable: - options.count += 1 - yield x - regen_iter(passthru(options.repo), err) - else: - queue = Queue(options.thread_count * 2) - kill = Event() - kill.clear() - def iter_queue(kill, qlist, timeout=0.25): - while not kill.isSet(): - try: - yield qlist.get(timeout=timeout) - except Empty: - continue - regen_threads = [ - Thread( - target=regen_iter, args=(iter_queue(kill, queue), err)) - for x in xrange(options.thread_count)] - out.write('starting %d threads' % (options.thread_count,)) - try: - for x in regen_threads: - x.start() - out.write('started') - # now we feed the queue. - for pkg in options.repo: - options.count += 1 - queue.put(pkg) - except Exception: - kill.set() - reclaim_threads(regen_threads, err) - raise - - # by now, queue is fed. reliable for our uses since the queue - # is only subtracted from. - while not queue.empty(): - sleep(.5) - kill.set() - reclaim_threads(regen_threads, err) - assert queue.empty() - out.write("finished %d nodes in in %.2f seconds" % (options.count, - time() - start_time)) - return 0 - -commandline_commands['regen'] = (RegenParser, regen_main) - - -class DigestParser(OptionParser): - - def __init__(self, **kwargs): - OptionParser.__init__( - self, description="generate digests for given atoms", **kwargs) - self.add_option('-t', '--type', type='choice', - choices=("manifest1", "manifest2", "both"), default="both", - help="type of manifest to generate (defaults to both). " - "valid values are: 'manifest1', 'manifest2', 'both'") - - def check_values(self, values, args): - values, args = OptionParser.check_values( - self, values, args) - - if not args: - self.error('Specify a repo') - repo = args.pop(0) - try: - values.repo = values.config.repo[repo] - except KeyError: - self.error("repo %r was not found, known repos-\n%s" % - (repo, format_seq(values.config.repo.keys()))) - - if values.type == "both": - values.type = ("manifest1", "manifest2") - else: - values.type = (values.type,) - - if not args: - self.error('Specify an atom') - values.atoms = [] - for arg in args: - try: - values.atoms.append(atom.atom(arg)) - except errors.MalformedAtom, e: - self.error(str(e)) - - return values, () - - -def digest_main(options, out, err): - """Write Manifests and digests""" - - for atom in options.atoms: - pkgs = options.repo.match(atom) - if not pkgs: - err.write('No matches for %s\n' % (options.atom,)) - return 1 - for pkg in pkgs: - if "manifest1" in options.type: - if options.debug: - out.write('Writing digest for %s:' % pkg.cpvstr) - location = pjoin(pkg.repo.location, pkg.key, "files", - "digest-%s-%s" % (pkg.versioned_atom.package, - pkg.versioned_atom.fullver)) - digest.serialize_digest(open(location, 'w'), pkg.fetchables) - if "manifest2" in options.type: - if options.debug: - out.write('Writing Manifest for %s:' % pkg.cpvstr) - digest.serialize_manifest("%s/%s" %(pkg.repo.location, pkg.key), - pkg.fetchables) - -# XXX: harring disabled this for 0.3. -# re-enable it when the bits update manifest. -#commandline_commands['digest'] = (DigestParser, digest_main) diff --git a/pkgcore/scripts/pmerge.py b/pkgcore/scripts/pmerge.py deleted file mode 100644 index adfb714..0000000 --- a/pkgcore/scripts/pmerge.py +++ /dev/null @@ -1,561 +0,0 @@ -# Copyright: 2006-2007 Brian Harring <ferringb@gmail.com> -# Copyright: 2006 Marien Zwart <marienz@gentoo.org> -# License: GPL2 - - -"""Mess with the resolver and vdb.""" - -from time import time - -from pkgcore.util import commandline, parserestrict, repo_utils -from pkgcore.ebuild import resolver -from pkgcore.repository import multiplex -from pkgcore.interfaces import observer, format -from pkgcore.pkgsets.glsa import KeyedAndRestriction -from pkgcore.ebuild.atom import atom - -from snakeoil import lists -from snakeoil.formatters import ObserverFormatter -from snakeoil.compatibility import any -from pkgcore.resolver.util import reduce_to_failures - -class OptionParser(commandline.OptionParser): - - def __init__(self, **kwargs): - commandline.OptionParser.__init__(self, description=__doc__, **kwargs) - self.add_option('--deep', '-D', action='store_true', - help='force the resolver to verify already installed dependencies') - self.add_option('--unmerge', '-C', action='store_true', - help='unmerge a package') - self.add_option('--clean', action='store_true', - help='remove installed packages that are not referenced by any ' - 'target packages/sets; defaults to -s world -s system if no targets' - ' are specified. Use with *caution*, this option used incorrectly ' - 'can render your system unusable. Implies --deep'), - self.add_option('--upgrade', '-u', action='store_true', - help='try to upgrade already installed packages/depencies') - self.add_option('--set', '-s', action='append', - help='specify a pkgset to use') - self.add_option('--ignore-failures', action='store_true', - help='ignore resolution failures') - self.add_option('--preload-vdb-state', action='store_true', - help=\ -"""enable preloading of the installed packages database -This causes the resolver to work with a complete graph, thus disallowing -actions that conflict with installed packages. If disabled, it's possible -for the requested action to conflict with already installed dependencies -that aren't involved in the graph of the requested operation""") - - self.add_option('--pretend', '-p', action='store_true', - help="do the resolution, but don't merge/fetch anything") - self.add_option('--ask', '-a', action='store_true', - help="do the resolution, but ask to merge/fetch anything") - self.add_option('--fetchonly', '-f', action='store_true', - help="do only the fetch steps of the resolved plan") - self.add_option('--ignore-cycles', '-i', action='store_true', - help=\ -"""ignore cycles if they're found to be unbreakable; -a depends on b, and b depends on a, with neither built is an example""") - - self.add_option('-B', '--with-built-depends', action='store_true', - default=False, - help="whether or not to process build depends for pkgs that " - "are already built; defaults to ignoring them"), - self.add_option('--nodeps', action='store_true', - help='disable dependency resolution') - self.add_option('--noreplace', action='store_false', - dest='replace', default=True, - help="don't reinstall target atoms if they're already installed") - self.add_option('--usepkg', '-k', action='store_true', - help="prefer to use binpkgs") - self.add_option('--usepkgonly', '-K', action='store_true', - help="use only built packages") - self.add_option('--empty', '-e', action='store_true', - help="force rebuilding of all involved packages, using installed " - "packages only to satisfy building the replacements") - self.add_option('--force', action='store_true', - dest='force', - help="force merging to a repo, regardless of if it's frozen") - self.add_option('--oneshot', '-o', '-1', action='store_true', - default=False, - help="do not record changes in the world file; if a set is " - "involved, defaults to forcing oneshot") - self.add_option( - '--formatter', '-F', action='callback', type='string', - callback=commandline.config_callback, - callback_args=('pmerge_formatter',), - help='which formatter to output --pretend or --ask output through.') - self.add_option('--domain', action='callback', type='string', - callback=commandline.config_callback, callback_args=('domain',), - help='specify which domain to use; else uses the "default" domain') - - def check_values(self, options, args): - options, args = commandline.OptionParser.check_values( - self, options, args) - options.targets = args - - # TODO this is rather boilerplate-ish, the commandline module - # should somehow do this for us. - if options.formatter is None: - options.formatter = options.config.get_default('pmerge_formatter') - if options.formatter is None: - self.error( - 'No default formatter found, fix your configuration ' - 'or pass --formatter (Valid formatters: %s)' % ( - ', '.join(options.config.pmerge_formatter),)) - - if options.domain is None: - options.domain = options.config.get_default('domain') - if options.domain is None: - self.error( - 'No default domain found, fix your configuration or pass ' - '--domain (valid domains: %s)' % - (', '.join(options.config.domain),)) - - if options.unmerge: - if options.set: - self.error("Using sets with -C probably isn't wise, aborting") - if options.upgrade: - self.error("Cannot upgrade and unmerge simultaneously") - if not options.targets: - self.error("You must provide at least one atom") - if options.clean: - self.error("Cannot use -C with --clean") - if options.clean: - if options.set or options.targets: - self.error("--clean currently has set/targets disabled; in " - "other words, accepts no args") - options.set = ['world', 'system'] - options.deep = True - if options.usepkgonly or options.usepkg: - self.error( - '--usepkg and --usepkgonly cannot be used with --clean') - elif options.usepkgonly and options.usepkg: - self.error('--usepkg is redundant when --usepkgonly is used') - if options.set: - options.replace = False - if not options.targets and not options.set: - self.error('Need at least one atom/set') - return options, () - -class AmbiguousQuery(parserestrict.ParseError): - def __init__(self, token, keys): - parserestrict.ParseError.__init__( - self, '%s: multiple matches (%s)' % (token, ', '.join(keys))) - self.token = token - self.keys = keys - -class NoMatches(parserestrict.ParseError): - def __init__(self, token): - parserestrict.ParseError.__init__(self, '%s: no matches' % (token,)) - -def parse_atom(token, repo, return_none=False): - """Use L{parserestrict.parse_match} to produce a single atom. - - This matches the restriction against the repo, raises - AmbiguousQuery if they belong to multiple cat/pkgs, returns an - atom otherwise. - - @param token: string to convert. - @param repo: L{pkgcore.repository.prototype.tree} instance to search in. - @param return_none: indicates if no matches raises or returns C{None} - - @return: an atom or C{None}. - """ - # XXX this should be in parserestrict in some form, perhaps. - restriction = parserestrict.parse_match(token) - key_matches = set(x.key for x in repo.itermatch(restriction)) - if not key_matches: - raise NoMatches(token) - elif len(key_matches) > 1: - raise AmbiguousQuery(token, sorted(key_matches)) - if isinstance(restriction, atom): - # atom is guranteed to be fine, since it's cat/pkg - return restriction - return KeyedAndRestriction(restriction, key=key_matches.pop()) - - -class Failure(ValueError): - """Raised internally to indicate an "expected" failure condition.""" - - -def unmerge(out, err, vdb, tokens, options, formatter, world_set=None): - """Unmerge tokens. hackish, should be rolled back into the resolver""" - all_matches = set() - for token in tokens: - # Catch restrictions matching across more than one category. - # Multiple matches in the same category are acceptable. - - # The point is that matching across more than one category is - # nearly always unintentional ("pmerge -C spork" without - # realising there are sporks in more than one category), but - # matching more than one cat/pkg is impossible without - # explicit wildcards. - restriction = parserestrict.parse_match(token) - matches = vdb.match(restriction) - if not matches: - raise Failure('Nothing matches %s' % (token,)) - categories = set(pkg.category for pkg in matches) - if len(categories) > 1: - raise parserestrict.ParseError( - '%s is in multiple categories (%s)' % ( - token, ', '.join(set(pkg.key for pkg in matches)))) - all_matches.update(matches) - - matches = sorted(all_matches) - out.write(out.bold, 'The following packages are to be unmerged:') - out.prefix = [out.bold, ' * ', out.reset] - for match in matches: - out.write(match.cpvstr) - out.prefix = [] - - repo_obs = observer.file_repo_observer(ObserverFormatter(out)) - - if options.pretend: - return - - if (options.ask and not - formatter.ask("Would you like to unmerge these packages?")): - return - return do_unmerge(options, out, err, vdb, matches, world_set, repo_obs) - -def do_unmerge(options, out, err, vdb, matches, world_set, repo_obs): - if vdb.frozen: - if options.force: - out.write( - out.fg('red'), out.bold, - 'warning: vdb is frozen, overriding') - vdb.frozen = False - else: - raise Failure('vdb is frozen') - - for idx, match in enumerate(matches): - out.write("removing %i of %i: %s" % (idx + 1, len(matches), match)) - out.title("%i/%i: %s" % (idx + 1, len(matches), match)) - op = vdb.uninstall(match, observer=repo_obs) - ret = op.finish() - if not ret: - if not options.ignore_failures: - raise Failure('failed unmerging %s' % (match,)) - out.write(out.fg('red'), 'failed unmerging ', match) - update_worldset(world_set, match, remove=True) - out.write("finished; removed %i packages" % len(matches)) - - -def get_pkgset(config, err, setname): - try: - return config.pkgset[setname] - except KeyError: - err.write('No set called %r!\nknown sets: %r' % - (setname, config.pkgset.keys())) - return None - -def display_failures(out, sequence, first_level=True): - sequence = iter(sequence) - frame = sequence.next() - if first_level: - # pops below need to exactly match. - out.first_prefix.extend((out.fg("red"), "!!!", out.reset)) - out.first_prefix.append(" ") - out.write("request %s, mode %s" % (frame.atom, frame.mode)) - for pkg, steps in sequence: - out.write("trying %s" % str(pkg.cpvstr)) - out.first_prefix.append(" ") - for step in steps: - if isinstance(step, list): - display_failures(out, step, False) - elif step[0] == 'reduce': - continue - elif step[0] == 'blocker': - out.write("blocker %s failed due to %s existing" % (step[1], - ', '.join(str(x) for x in step[2]))) - elif step[0] == 'cycle': - out.write("%s cycle on %s: %s" % (step[2].mode, step[2].atom, step[3])) - elif step[0] == 'viable' and not step[1]: - out.write("%s: failed %s" % (step[3], step[4])) - else: - out.write(step) - out.first_prefix.pop() - out.first_prefix.pop() - if first_level: - [out.first_prefix.pop() for x in (1,2,3)] - -def update_worldset(world_set, pkg, remove=False): - if world_set is None: - return - if remove: - try: - world_set.remove(pkg) - except KeyError: - # nothing to remove, thus skip the flush - return - else: - world_set.add(pkg) - world_set.flush() - -def main(options, out, err): - config = options.config - if options.debug: - resolver.plan.limiters.add(None) - - domain = options.domain - vdb = domain.all_vdbs - - formatter = options.formatter(out=out, err=err, - use_expand=domain.use_expand, - use_expand_hidden=domain.use_expand_hidden) - - # This mode does not care about sets and packages so bypass all that. - if options.unmerge: - world_set = None - if not options.oneshot: - world_set = get_pkgset(config, err, "world") - if world_set is None: - err.write("Disable world updating via --oneshot, or fix your " - "configuration") - return 1 - try: - unmerge( - out, err, vdb, options.targets, options, formatter, world_set) - except (parserestrict.ParseError, Failure), e: - out.error(str(e)) - return 1 - return - - all_repos = domain.all_repos - repos = list(all_repos.trees) - if options.usepkgonly or options.usepkg: - if options.usepkgonly: - repos = [ - repo for repo in all_repos.trees - if getattr(repo, 'format_magic', None) != 'ebuild_src'] - else: - repos = [ - repo for repo in all_repos.trees - if getattr(repo, 'format_magic', None) == 'ebuild_built'] + [ - repo for repo in all_repos.trees - if getattr(repo, 'format_magic', None) != 'ebuild_built'] - all_repos = multiplex.tree(*repos) - - atoms = [] - for setname in options.set: - pkgset = get_pkgset(config, err, setname) - if pkgset is None: - return 1 - l = list(pkgset) - if not l: - out.write("skipping set %s: set is empty, nothing to update" % setname) - else: - atoms.extend(l) - - for token in options.targets: - try: - a = parse_atom(token, all_repos, return_none=True) - except parserestrict.ParseError, e: - out.error(str(e)) - return 1 - if a is None: - if token in config.pkgset: - out.error( - 'No package matches %r, but there is a set with ' - 'that name. Use -s to specify a set.' % (token,)) - return 2 - elif not options.ignore_failures: - out.error('No matches for %r; ignoring it' % token) - else: - return -1 - else: - atoms.append(a) - - if not atoms: - out.error('No targets specified; nothing to do') - return 1 - - atoms = lists.stable_unique(atoms) - - world_set = None - if (not options.set or options.clean) and not options.oneshot: - world_set = get_pkgset(config, err, 'world') - if world_set is None: - err.write("Disable world updating via --oneshot, or fix your " - "configuration") - return 1 - - if options.upgrade: - resolver_kls = resolver.upgrade_resolver - else: - resolver_kls = resolver.min_install_resolver - - extra_kwargs = {} - if options.empty: - extra_kwargs['resolver_cls'] = resolver.empty_tree_merge_plan - if options.debug: - extra_kwargs['debug'] = True - - resolver_inst = resolver_kls( - vdb, repos, verify_vdb=options.deep, nodeps=options.nodeps, - drop_cycles=options.ignore_cycles, force_replacement=options.replace, - process_built_depends=options.with_built_depends, - **extra_kwargs) - - if options.preload_vdb_state: - out.write(out.bold, ' * ', out.reset, 'Preloading vdb... ') - vdb_time = time() - resolver_inst.load_vdb_state() - vdb_time = time() - vdb_time - else: - vdb_time = 0.0 - - failures = [] - resolve_time = time() - out.write(out.bold, ' * ', out.reset, 'Resolving...') - out.title('Resolving...') - for restrict in atoms: - ret = resolver_inst.add_atom(restrict) - if ret: - out.error('resolution failed') - just_failures = reduce_to_failures(ret[1]) - display_failures(out, just_failures) - failures.append(restrict) - if not options.ignore_failures: - break - resolve_time = time() - resolve_time - if failures: - out.write() - out.write('Failures encountered:') - for restrict in failures: - out.error("failed '%s'" % (restrict,)) - out.write('potentials:') - match_count = 0 - for r in repo_utils.get_raw_repos(repos): - l = r.match(restrict) - if l: - out.write( - "repo %s: [ %s ]" % (r, ", ".join(str(x) for x in l))) - match_count += len(l) - if not match_count: - out.write("No matches found in %s" % (repos,)) - out.write() - if not options.ignore_failures: - return 1 - - if options.clean: - out.write(out.bold, ' * ', out.reset, 'Packages to be removed:') - vset = set(vdb) - len_vset = len(vset) - vset.difference_update(y.pkg for y in - resolver_inst.state.iter_ops(True)) - wipes = sorted(x for x in vset if x.package_is_real) - for x in wipes: - out.write("Remove %s" % x) - out.write() - if wipes: - out.write("removing %i packages of %i installed, %0.2f%%." % - (len(wipes), len_vset, 100*(len(wipes)/float(len_vset)))) - else: - out.write("no packages to remove") - if options.pretend: - return 0 - if options.ask: - if not formatter.ask("Do you wish to proceed?", default_answer=False): - return 1 - out.write() - repo_obs = observer.file_repo_observer(ObserverFormatter(out)) - do_unmerge(options, out, err, vdb, wipes, world_set, repo_obs) - return 0 - - changes = list(x for x in resolver_inst.state.iter_ops() - if x.pkg.package_is_real) - - if options.ask or options.pretend: - for op in changes: - formatter.format(op) - formatter.end() - - - if vdb_time: - out.write(out.bold, 'Took %.2f' % (vdb_time,), out.reset, - ' seconds to preload vdb state') - if options.pretend: - return - - if (options.ask and not - formatter.ask("Would you like to merge these packages?")): - return - - build_obs = observer.file_build_observer(ObserverFormatter(out)) - repo_obs = observer.file_repo_observer(ObserverFormatter(out)) - - change_count = len(changes) - for count, op in enumerate(changes): - out.write("Processing %i of %i: %s" % (count + 1, change_count, - op.pkg.cpvstr)) - out.title("%i/%i: %s" % (count + 1, change_count, op.pkg.cpvstr)) - if op.desc != "remove": - if not options.fetchonly and options.debug: - out.write("Forcing a clean of workdir") - buildop = op.pkg.build(observer=build_obs, clean=True) - if options.fetchonly: - out.write("\n%i files required-" % len(op.pkg.fetchables)) - try: - ret = buildop.fetch() - except (SystemExit, KeyboardInterrupt): - raise - except Exception, e: - ret = e - if ret != True: - out.error("got %s for a phase execution for %s" % (ret, op.pkg)) - if not options.ignore_failures: - return 1 - buildop.cleanup() - del buildop, ret - continue - - ret = None - try: - built_pkg = buildop.finalize() - if built_pkg is False: - ret = built_pkg - except format.errors, e: - ret = e - if ret is not None: - out.error("Failed to build %s: %s" % (op.pkg, ret)) - if not options.ignore_failures: - return 1 - continue - - out.write() - if op.desc == "replace": - if op.old_pkg == op.pkg: - out.write(">>> Reinstalling %s" % (built_pkg.cpvstr)) - else: - out.write(">>> Replacing %s with %s" % ( - op.old_pkg.cpvstr, built_pkg.cpvstr)) - i = vdb.replace(op.old_pkg, built_pkg, observer=repo_obs) - - else: - out.write(">>> Installing %s" % built_pkg.cpvstr) - i = vdb.install(built_pkg, observer=repo_obs) - - # force this explicitly- can hold onto a helluva lot more - # then we would like. - del built_pkg - else: - out.write(">>> Removing %s" % op.pkg.cpvstr) - i = vdb.uninstall(op.pkg, observer=repo_obs) - ret = i.finish() - if ret != True: - out.error("got %s for a phase execution for %s" % (ret, op.pkg)) - if not options.ignore_failures: - return 1 - buildop.cleanup() - if world_set: - if op.desc == "remove": - out.write('>>> Removing %s from world file' % op.pkg.cpvstr) - update_worldset(world_set, op.pkg, remove=True) - elif any(x.match(op.pkg) for x in atoms): - if not options.upgrade: - out.write('>>> Adding %s to world file' % op.pkg.cpvstr) - update_worldset(world_set, op.pkg) - out.write("finished") - return 0 diff --git a/pkgcore/scripts/pplugincache.py b/pkgcore/scripts/pplugincache.py deleted file mode 100644 index ce5cdc7..0000000 --- a/pkgcore/scripts/pplugincache.py +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright: 2006 Marien Zwart <marienz@gentoo.org> -# License: GPL2 - - -"""Update the plugin cache.""" - - -from pkgcore.util import commandline -from pkgcore import plugin -from snakeoil import modules - -class OptionParser(commandline.OptionParser): - - def __init__(self, **kwargs): - commandline.OptionParser.__init__( - self, description=__doc__, usage='%prog [packages]', **kwargs) - - def check_values(self, values, args): - """Sanity check and postprocess after parsing.""" - values, args = commandline.OptionParser.check_values( - self, values, args) - if not args: - args = ['pkgcore.plugins'] - values.packages = [] - for arg in args: - try: - package = modules.load_module(arg) - except modules.FailedImport, e: - self.error('Failed to import %s (%s)' % (arg, e)) - if not getattr(package, '__path__', False): - self.error('%s is not a package' % (arg,)) - values.packages.append(package) - return values, () - - -def main(options, out, err): - """Update caches.""" - for package in options.packages: - out.write('Updating cache for %s...' % (package.__name__,)) - plugin.initialize_cache(package) diff --git a/pkgcore/scripts/pquery.py b/pkgcore/scripts/pquery.py deleted file mode 100644 index 592609c..0000000 --- a/pkgcore/scripts/pquery.py +++ /dev/null @@ -1,882 +0,0 @@ -# Copyright: 2006 Marien Zwart <marienz@gentoo.org> -# License: GPL2 -# Based on pquery by Brian Harring <ferringb@gmail.com> - - -"""Extract information from repositories.""" - - - -from pkgcore.restrictions import packages, values, boolean, restriction -from pkgcore.ebuild import conditionals, atom -from pkgcore.util import ( - commandline, repo_utils, parserestrict, packages as pkgutils) - -# ordering here matters; pkgcore does a trick to commandline to avoid the -# heavy inspect loadup hit. -import optparse - - -# To add a new restriction you have to do the following: -# - add a parse function for it here. -# - add the parse function to the PARSE_FUNCS dict. -# - add an optparse option using the name you used in the dict as -# both the typename and the long option name. - -def parse_revdep(value): - """Value should be an atom, packages with deps intersecting that match.""" - try: - targetatom = atom.atom(value) - except atom.MalformedAtom, e: - raise parserestrict.ParseError(str(e)) - val_restrict = values.FlatteningRestriction( - atom.atom, - values.AnyMatch(values.FunctionRestriction(targetatom.intersects))) - return packages.OrRestriction(finalize=True, *list( - packages.PackageRestriction(dep, val_restrict) - for dep in ('depends', 'rdepends', 'post_rdepends'))) - -def parse_description(value): - """Value is used as a regexp matching description or longdescription.""" - matcher = values.StrRegex(value, case_sensitive=False) - return packages.OrRestriction(finalize=True, *list( - packages.PackageRestriction(attr, matcher) - for attr in ('description', 'longdescription'))) - -def parse_ownsre(value): - """Value is a regexp matched against the string form of an fs object. - - This means the object kind is prepended to the path the regexp has - to match. - """ - return packages.PackageRestriction( - 'contents', values.AnyMatch(values.GetAttrRestriction( - 'location', values.StrRegex(value)))) - - -class DataSourceRestriction(values.base): - - """Turn a data_source into a line iterator and apply a restriction.""" - - def __init__(self, childrestriction, **kwargs): - values.base.__init__(self, **kwargs) - self.restriction = childrestriction - - def __str__(self): - return 'DataSourceRestriction: %s negate=%s' % ( - self.restriction, self.negate) - - def __repr__(self): - if self.negate: - string = '<%s restriction=%r negate @%#8x>' - else: - string = '<%s restriction=%r @%#8x>' - return string % (self.__class__.__name__, self.restriction, id(self)) - - def match(self, value): - return self.restriction.match(iter(value.get_fileobj())) ^ self.negate - - __hash__ = object.__hash__ - - -def parse_envmatch(value): - """Apply a regexp to the environment.""" - return packages.PackageRestriction( - 'environment', DataSourceRestriction(values.AnyMatch( - values.StrRegex(value)))) - - -def parse_maintainer_email(value): - """ - Case insensitive Regex match on the email bit of metadata.xml's - maintainer data. - """ - return packages.PackageRestriction( - 'maintainers', values.AnyMatch(values.GetAttrRestriction( - 'email', values.StrRegex(value.lower(), - case_sensitive=False)))) - -def parse_maintainer_name(value): - """ - Case insensitive Regex match on the name bit of metadata.xml's - maintainer data. - """ - return packages.PackageRestriction( - 'maintainers', values.AnyMatch(values.GetAttrRestriction( - 'name', values.StrRegex(value.lower(), - case_sensitive=False)))) - -def parse_maintainer(value): - """ - Case insensitive Regex match on the combined 'name <email>' bit of - metadata.xml's maintainer data. - """ - return packages.PackageRestriction( - 'maintainers', values.AnyMatch( - values.UnicodeConversion( - values.StrRegex(value.lower(), - case_sensitive=False)))) - - -def parse_expression(string): - """Convert a string to a restriction object using pyparsing.""" - # Two reasons to delay this import: we want to deal if it is - # not there and the import is slow (needs to compile a bunch - # of regexps). - try: - import pyparsing as pyp - except ImportError: - raise parserestrict.ParseError('pyparsing is not installed.') - - grammar = getattr(parse_expression, 'grammar', None) - if grammar is None: - - anystring = pyp.quotedString.copy().setParseAction(pyp.removeQuotes) - anystring |= pyp.Word(pyp.alphanums + ',') - - def funcall(name, parser): - """Create a pyparsing expression from a name and parse func.""" - # This function cannot be inlined below: we use its scope to - # "store" the parser function. If we store the parser function - # as default argument to the _parse function pyparsing passes - # different arguments (it detects the number of arguments the - # function takes). - result = (pyp.Suppress('%s(' % (name,)) + anystring + - pyp.Suppress(')')) - def _parse(tokens): - return parser(tokens[0]) - result.setParseAction(_parse) - return result - - - boolcall = pyp.Forward() - expr = boolcall - for name, func in PARSE_FUNCS.iteritems(): - expr |= funcall(name, func) - - andcall = (pyp.Suppress(pyp.CaselessLiteral('and') + '(') + - pyp.delimitedList(expr) + pyp.Suppress(')')) - def _parse_and(tokens): - return packages.AndRestriction(*tokens) - andcall.setParseAction(_parse_and) - - orcall = (pyp.Suppress(pyp.CaselessLiteral('or') + '(') + - pyp.delimitedList(expr) + pyp.Suppress(')')) - def _parse_or(tokens): - return packages.OrRestriction(*tokens) - orcall.setParseAction(_parse_or) - - notcall = (pyp.Suppress(pyp.CaselessLiteral('not') + '(') + expr + - pyp.Suppress(')')) - def _parse_not(tokens): - return restriction.Negate(tokens[0]) - notcall.setParseAction(_parse_not) - - # "Statement seems to have no effect" - # pylint: disable-msg=W0104 - boolcall << (notcall | andcall | orcall) - - # This forces a match on the entire thing, without it trailing - # unparsed data is ignored. - grammar = pyp.stringStart + expr + pyp.stringEnd - - # grammar.validate() - - parse_expression.grammar = grammar - - try: - return grammar.parseString(string)[0] - except pyp.ParseException, e: - raise parserestrict.ParseError(e.msg) - - -PARSE_FUNCS = { - 'restrict_revdep': parse_revdep, - 'description': parse_description, - 'ownsre': parse_ownsre, - 'environment': parse_envmatch, - 'expr': parse_expression, - 'maintainer': parse_maintainer, - 'maintainer_name': parse_maintainer_name, - 'maintainer_email': parse_maintainer_email, - } - -# This is not just a blind "update" because we really need a config -# option for everything in this dict (so parserestrict growing parsers -# would break us). -for _name in ['match']: - PARSE_FUNCS[_name] = parserestrict.parse_funcs[_name] - -for _name, _attr in [ - ('herd', 'herds'), - ('license', 'license'), - ('hasuse', 'iuse'), - ('owns', 'contents'), - ]: - PARSE_FUNCS[_name] = parserestrict.comma_separated_containment(_attr) - -del _name, _attr - - -def optparse_type(parsefunc): - """Wrap a parsefunc shared with the expression-style code for optparse.""" - def _typecheck(option, opt, value): - try: - return parsefunc(value) - except parserestrict.ParseError, e: - raise optparse.OptionValueError('option %s: %s' % (opt, e)) - return _typecheck - - -def atom_type(option, opt, value): - try: - return atom.atom(value) - except atom.MalformedAtom, e: - raise optparse.OptionValueError('option %s: %s' % (opt, e)) - - -extras = dict((parser_name, optparse_type(parser_func)) - for parser_name, parser_func in PARSE_FUNCS.iteritems()) -extras['atom'] = atom_type - -class Option(commandline.Option): - """C{optparse.Option} subclass supporting our custom types.""" - TYPES = optparse.Option.TYPES + tuple(extras.keys()) - # Copy the original dict - TYPE_CHECKER = dict(optparse.Option.TYPE_CHECKER) - TYPE_CHECKER.update(extras) - - -def append_const_callback(option, opt_str, value, parser, const): - """Callback version of python 2.5's append_const action.""" - parser.values.ensure_value(option.dest, []).append(const) - - -def revdep_callback(option, opt_str, value, parser): - try: - parser.values.ensure_value('restrict_revdep', []).append( - parse_revdep(value)) - parser.values.ensure_value('print_revdep', []).append(atom.atom(value)) - except (parserestrict.ParseError, atom.MalformedAtom), e: - raise optparse.OptionValueError('option %s: %s' % (opt_str, e)) - - -class OptionParser(commandline.OptionParser): - - """Option parser with custom option postprocessing and validation.""" - - def __init__(self, **kwargs): - commandline.OptionParser.__init__( - self, description=__doc__, option_class=Option, **kwargs) - - self.set_default('pkgset', []) - self.set_default('restrict', []) - - self.add_option('--domain', action='callback', type='string', - callback=commandline.config_callback, - callback_args=('domain',), - help='domain name to use (default used if omitted).') - self.add_option('--repo', action='callback', type='string', - callback=commandline.config_callback, - callback_args=('repo',), - help='repo to use (default from domain if omitted).') - self.add_option('--early-out', action='store_true', dest='earlyout', - help='stop when first match is found.') - self.add_option('--no-version', '-n', action='store_true', - dest='noversion', - help='collapse multiple matching versions together') - self.add_option('--min', action='store_true', - help='show only the lowest version for each package.') - self.add_option('--max', action='store_true', - help='show only the highest version for each package.') - - repo = self.add_option_group('Source repo') - repo.add_option('--raw', action='store_true', - help='Without this switch your configuration affects ' - 'what packages are visible (through masking) and what ' - 'USE flags are applied to depends and fetchables. ' - "With this switch your configuration values aren't " - 'used and you see the "raw" repository data.') - repo.add_option( - '--virtuals', action='store', choices=('only', 'disable'), - help='arg "only" for only matching virtuals, "disable" to not ' - 'match virtuals at all. Default is to match everything.') - repo.add_option('--vdb', action='store_true', - help='match only vdb (installed) packages.') - repo.add_option('--all-repos', action='store_true', - help='search all repos, vdb included') - - restrict = self.add_option_group( - 'Package matching', - 'Each option specifies a restriction packages must match. ' - 'Specifying the same option twice means "or" unless stated ' - 'otherwise. Specifying multiple types of restrictions means "and" ' - 'unless stated otherwise.') - restrict.add_option('--all', action='callback', - callback=append_const_callback, - callback_args=(packages.AlwaysTrue,), - dest='restrict', - help='Match all packages (equivalent to -m "*")') - restrict.add_option( - '--match', '-m', action='append', type='match', - help='Glob-like match on category/package-version.') - restrict.add_option('--has-use', action='append', type='hasuse', - dest='hasuse', - help='Exact string match on a USE flag.') - restrict.add_option( - '--revdep', action='callback', callback=revdep_callback, - type='string', - help='shorthand for --restrict-revdep atom --print-revdep atom. ' - '--print-revdep is slow, use just --restrict-revdep if you just ' - 'need a list.') - restrict.add_option( - '--restrict-revdep', action='append', type='restrict_revdep', - help='Dependency on an atom.') - restrict.add_option('--description', '-S', action='append', - type='description', - help='regexp search on description and longdescription.') - restrict.add_option('--herd', action='append', type='herd', - help='exact match on a herd.') - restrict.add_option('--license', action='append', type='license', - help='exact match on a license.') - restrict.add_option('--owns', action='append', type='owns', - help='exact match on an owned file/dir.') - restrict.add_option( - '--owns-re', action='append', type='ownsre', dest='ownsre', - help='like "owns" but using a regexp for matching.') - restrict.add_option('--maintainer', action='append', type='maintainer', - help='comma-separated list of regexes to search for ' - 'maintainers.') - restrict.add_option('--maintainer-name', action='append', type='maintainer_name', - help='comma-separated list of maintainer name regexes ' - 'to search for.') - restrict.add_option('--maintainer-email', action='append', type='maintainer_email', - help='comma-separated list of maintainer email regexes ' - 'to search for.') - restrict.add_option( - '--environment', action='append', type='environment', - help='regexp search in environment.bz2.') - restrict.add_option( - '--expr', action='append', type='expr', - help='Boolean combinations of other restrictions, like ' - '\'and(not(herd("python")), match("dev-python/*"))\'. ' - 'WARNING: currently not completely reliable.', - long_help='Boolean combinations of other restrictions, like ' - '``and(not(herd("python")), match("dev-python/*"))``. ' - '*WARNING*: currently not completely reliable.' - ) - # XXX fix the negate stuff and remove that warning. - restrict.add_option( - '--pkgset', action='callback', type='string', - callback=commandline.config_append_callback, - callback_args=('pkgset',), - help='is inside a named set of packages (like "world").') - - printable_attrs = ('rdepends', 'depends', 'post_rdepends', 'provides', - 'use', 'iuse', 'description', 'longdescription', - 'herds', 'license', 'uris', 'files', - 'slot', 'maintainers', 'restrict', 'repo', - 'alldepends', 'path', 'environment', 'keywords', - 'homepage', 'fetchables') - - output = self.add_option_group('Output formatting') - output.add_option( - '--cpv', action='store_true', - help='Print the category/package-version. This is done ' - 'by default, this option re-enables this if another ' - 'output option (like --contents) disabled it.') - output.add_option('--atom', '-a', action='store_true', - help='print =cat/pkg-3 instead of cat/pkg-3. ' - 'Implies --cpv, has no effect with --no-version') - output.add_option('--attr', action='append', choices=printable_attrs, - help="Print this attribute's value (can be specified more than " - "once). --attr=help will get you the list of valid attrs.") - output.add_option('--one-attr', choices=printable_attrs, - help="Print one attribute. Suppresses other output.") - output.add_option('--force-attr', action='append', dest='attr', - help='Like --attr but accepts any string as ' - 'attribute name instead of only explicitly ' - 'supported names.') - output.add_option('--force-one-attr', - help='Like --oneattr but accepts any string as ' - 'attribute name instead of only explicitly ' - 'supported names.') - output.add_option( - '--contents', action='store_true', - help='list files owned by the package. Implies --vdb.') - output.add_option('--verbose', '-v', action='store_true', - help='human-readable multi-line output per package') - output.add_option('--highlight-dep', action='append', type='atom', - help='highlight dependencies matching this atom') - output.add_option( - '--blame', action='store_true', - help='shorthand for --attr maintainers --attr herds') - output.add_option( - '--print-revdep', type='atom', action='append', - help='print what condition(s) trigger a dep.') - - def check_values(self, values, args): - """Sanity check and postprocess after parsing.""" - vals, args = commandline.OptionParser.check_values(self, values, args) - # Interpret args with parens in them as --expr additions, the - # rest as --match additions (since parens are invalid in --match). - try: - for arg in args: - if '(' in arg: - vals.expr.append(parse_expression(arg)) - else: - vals.match.append(parserestrict.parse_match(arg)) - except parserestrict.ParseError, e: - self.error(str(e)) - - # TODO come up with something better than "match" for this. - for highlight in vals.highlight_dep: - if not isinstance(highlight, atom.atom): - self.error('highlight-dep must be an atom') - - if vals.contents or vals.owns or vals.ownsre: - vals.vdb = True - - if vals.atom: - vals.cpv = True - - if vals.noversion: - if vals.contents: - self.error( - 'both --no-version and --contents does not make sense.') - if vals.min or vals.max: - self.error( - '--no-version with --min or --max does not make sense.') - if vals.print_revdep: - self.error( - '--print-revdep with --no-version does not make sense.') - - if vals.blame: - vals.attr.extend(['herds', 'maintainers']) - - if 'alldepends' in vals.attr: - vals.attr.remove('alldepends') - vals.attr.extend(['depends', 'rdepends', 'post_rdepends']) - - if vals.verbose: - # slice assignment to an empty range; behaves as an insertion. - vals.attr[0:0] = ['repo', 'description', 'homepage'] - - if vals.force_one_attr: - if vals.one_attr: - self.error( - '--one-attr and --force-one-attr are mutually exclusive.') - vals.one_attr = vals.force_one_attr - - if vals.one_attr and vals.print_revdep: - self.error( - '--print-revdep with --force-one-attr or --one-attr does not ' - 'make sense.') - - # Build up a restriction. - for attr in PARSE_FUNCS: - val = getattr(vals, attr) - if len(val) == 1: - # Omit the boolean. - vals.restrict.append(val[0]) - elif val: - vals.restrict.append( - packages.OrRestriction(finalize=True, *val)) - - all_atoms = [] - for pkgset in vals.pkgset: - atoms = list(pkgset) - if not atoms: - # This is currently an error because I am unsure what - # it should do. - self.error('Cannot use empty pkgsets') - all_atoms.extend(atoms) - if all_atoms: - vals.restrict.append(packages.OrRestriction(finalize=True, - *all_atoms)) - - if not vals.restrict: - self.error('No restrictions specified.') - - if len(vals.restrict) == 1: - # Single restriction, omit the AndRestriction for a bit of speed - vals.restrict = vals.restrict[0] - else: - # "And" them all together - vals.restrict = packages.AndRestriction(*vals.restrict) - - if vals.repo and (vals.vdb or vals.all_repos): - self.error( - '--repo with --vdb, --all-repos makes no sense') - - # Get a domain object if needed. - if vals.domain is None and ( - vals.verbose or vals.noversion or not vals.repo): - vals.domain = vals.config.get_default('domain') - if vals.domain is None: - self.error( - 'No default domain found, fix your configuration ' - 'or pass --domain (Valid domains: %s)' % ( - ', '.join(vals.config.domain),)) - - domain = vals.domain - # Get the vdb if we need it. - if vals.verbose and vals.noversion: - vals.vdbs = domain.vdb - else: - vals.vdbs = None - # Get repo(s) to operate on. - if vals.vdb: - vals.repos = domain.vdb - elif vals.all_repos: - vals.repos = domain.repos + domain.vdb - elif vals.repo: - vals.repos = [vals.repo] - else: - vals.repos = domain.repos - if vals.raw or vals.virtuals: - vals.repos = repo_utils.get_raw_repos(vals.repos) - if vals.virtuals: - vals.repos = repo_utils.get_virtual_repos( - vals.repos, vals.virtuals == 'only') - - return vals, () - - -def stringify_attr(config, pkg, attr): - """Grab a package attr and convert it to a string.""" - # config is currently unused but may affect display in the future. - if attr in ('files', 'uris'): - data = getattr(pkg, 'fetchables', None) - if data is None: - return 'MISSING' - if attr == 'files': - def _format(node): - return node.filename - else: - def _format(node): - return ' '.join(node.uri or ()) - return conditionals.stringify_boolean(data, _format) - - if attr == 'use': - # Combine a list of all enabled (including irrelevant) and all - # available flags into a "enabled -disabled" style string. - use = set(getattr(pkg, 'use', set())) - iuse = set(getattr(pkg, 'iuse', set())) - result = sorted(iuse & use) + sorted('-' + val for val in (iuse - use)) - return ' '.join(result) - - # TODO: is a missing or None attr an error? - value = getattr(pkg, attr, None) - if value is None: - return 'MISSING' - - if attr in ('herds', 'iuse', 'maintainers'): - return ' '.join(sorted(unicode(v) for v in value)) - if attr == 'keywords': - return ' '.join(sorted(value, key=lambda x:x.lstrip("~"))) - if attr == 'environment': - return ''.join(value.get_fileobj()) - if attr == 'repo': - return str(getattr(value, 'repo_id', 'no repo id')) - return str(value) - - -def _default_formatter(out, node): - out.write(node, autoline=False) - return False - - -def format_depends(out, node, func=_default_formatter): - """Pretty-print a depset to a formatter. - - @param out: formatter. - @param node: a L{conditionals.DepSet}. - @param func: callable taking a formatter and a depset payload. - If it can format its value in a single line it should do that - without writing a newline and return C{False}. - If it needs multiple lines it should first write a newline, not write - a terminating newline, and return C{True}. - @returns: The same kind of boolean func should return. - """ - oldwrap = out.wrap - out.wrap = False - try: - # Do this first since if it is a DepSet it is also an - # AndRestriction (DepSet subclasses that). - if isinstance(node, conditionals.DepSet): - if not node.restrictions: - return False - if len(node.restrictions) == 1: - return format_depends(out, node.restrictions[0], func) - out.write() - for child in node.restrictions[:-1]: - format_depends(out, child, func) - out.write() - format_depends(out, node.restrictions[-1], func) - return True - - prefix = None - if isinstance(node, boolean.OrRestriction): - prefix = '|| (' - children = node.restrictions - elif (isinstance(node, boolean.AndRestriction) and not - isinstance(node, atom.atom)): - prefix = '(' - children = node.restrictions - elif isinstance(node, packages.Conditional): - assert len(node.restriction.vals) == 1 - prefix = '%s%s? (' % (node.restriction.negate and '!' or '', - list(node.restriction.vals)[0]) - children = node.payload - - if prefix: - children = list(children) - if len(children) == 1: - out.write(prefix, ' ', autoline=False) - out.first_prefix.append(' ') - newline = format_depends(out, children[0], func) - out.first_prefix.pop() - if newline: - out.write() - out.write(')') - return True - else: - out.write(' )', autoline=False) - return False - else: - out.write(prefix) - out.first_prefix.append(' ') - for child in children: - format_depends(out, child, func) - out.write() - out.first_prefix.pop() - out.write(')', autoline=False) - return True - else: - return func(out, node) - finally: - out.wrap = oldwrap - -def format_attr(config, out, pkg, attr): - """Grab a package attr and print it through a formatter.""" - # config is currently unused but may affect display in the future. - if attr in ('depends', 'rdepends', 'post_rdepends', 'restrict'): - data = getattr(pkg, attr, None) - if data is None: - out.write('MISSING') - else: - out.first_prefix.append(' ') - if config.highlight_dep: - def _format(out, node): - for highlight in config.highlight_dep: - if highlight.intersects(node): - out.write(out.bold, out.fg('cyan'), node, - autoline=False) - return - out.write(node, autoline=False) - format_depends(out, data, _format) - else: - format_depends(out, data) - out.first_prefix.pop() - out.write() - elif attr in ('files', 'uris'): - data = getattr(pkg, 'fetchables', None) - if data is None: - out.write('MISSING') - return - if attr == 'files': - def _format(out, node): - out.write(node.filename, autoline=False) - else: - def _format(out, node): - uris = list(node.uri) - if not uris: - return False - if len(uris) == 1: - out.write(uris[0], autoline=False) - return False - out.write('|| (') - out.first_prefix.append(' ') - for uri in uris: - out.write(uri) - out.first_prefix.pop() - out.write(')', autoline=False) - return True - out.first_prefix.append(' ') - format_depends(out, data, _format) - out.first_prefix.pop() - out.write() - else: - out.write(stringify_attr(config, pkg, attr)) - - -def print_package(options, out, err, pkg): - """Print a package.""" - if options.verbose: - green = out.fg('green') - out.write(out.bold, green, ' * ', out.fg(), pkg.cpvstr) - out.wrap = True - out.later_prefix = [' '] - for attr in options.attr: - out.write(green, ' %s: ' % (attr,), out.fg(), autoline=False) - format_attr(options, out, pkg, attr) - for revdep in options.print_revdep: - for name in ('depends', 'rdepends', 'post_rdepends'): - depset = getattr(pkg, name) - find_cond = getattr(depset, 'find_cond_nodes', None) - if find_cond is None: - out.write( - green, ' revdep: ', out.fg(), name, ' on ', - str(revdep)) - continue - for key, restricts in depset.find_cond_nodes( - depset.restrictions, True): - if not restricts and key.intersects(revdep): - out.write( - green, ' revdep: ', out.fg(), name, ' on ', - autoline=False) - if key == revdep: - # this is never reached... - out.write(out.bold, str(revdep)) - else: - out.write( - str(revdep), ' through dep ', out.bold, - str(key)) - for key, restricts in depset.node_conds.iteritems(): - if key.intersects(revdep): - out.write( - green, ' revdep: ', out.fg(), name, ' on ', - autoline=False) - if key == revdep: - out.write( - out.bold, str(revdep), out.reset, - autoline=False) - else: - out.write( - str(revdep), ' through dep ', out.bold, - str(key), out.reset, autoline=False) - out.write(' if USE matches one of:') - for r in restricts: - out.write(' ', str(r)) - out.write() - out.later_prefix = [] - out.wrap = False - elif options.one_attr: - if options.atom: - out.write('=', autoline=False) - if options.atom or options.cpv: - out.write(pkg.cpvstr, ':', autoline=False) - out.write(stringify_attr(options, pkg, options.one_attr)) - else: - printed_something = False - out.autoline = False - if (not options.contents) or options.cpv: - printed_something = True - if options.atom: - out.write('=') - out.write(pkg.cpvstr) - for attr in options.attr: - if printed_something: - out.write(' ') - printed_something = True - out.write('%s="%s"' % (attr, stringify_attr(options, pkg, attr))) - for revdep in options.print_revdep: - for name in ('depends', 'rdepends', 'post_rdepends'): - depset = getattr(pkg, name) - if getattr(depset, 'find_cond_nodes', None) is None: - # TODO maybe be smarter here? (this code is - # triggered by virtuals currently). - out.write(' %s on %s' % (name, revdep)) - continue - for key, restricts in depset.find_cond_nodes( - depset.restrictions, True): - if not restricts and key.intersects(revdep): - out.write(' %s on %s through %s' % (name, revdep, key)) - for key, restricts in depset.node_conds.iteritems(): - if key.intersects(revdep): - out.write(' %s on %s through %s if USE %s,' % ( - name, revdep, key, ' or '.join( - str(r) for r in restricts))) - # If we printed anything at all print the newline now - out.autoline = True - if printed_something: - out.write() - - if options.contents: - for location in sorted(obj.location - for obj in getattr(pkg, 'contents', ())): - out.write(location) - -def print_packages_noversion(options, out, err, pkgs): - """Print a summary of all versions for a single package.""" - if options.verbose: - green = out.fg('green') - out.write(out.bold, green, ' * ', out.fg(), pkgs[0].key) - out.wrap = True - out.later_prefix = [' '] - versions = ' '.join(pkg.fullver for pkg in sorted(pkgs)) - out.write(green, ' versions: ', out.fg(), versions) - # If we are already matching on all repos we do not need to duplicate. - if not (options.vdb or options.all_repos): - versions = sorted( - pkg.fullver for vdb in options.vdbs - for pkg in vdb.itermatch(pkgs[0].unversioned_atom)) - out.write(green, ' installed: ', out.fg(), ' '.join(versions)) - for attr in options.attr: - out.write(green, ' %s: ' % (attr,), out.fg(), - stringify_attr(options, pkgs[-1], attr)) - out.write() - out.wrap = False - out.later_prefix = [] - elif options.one_attr: - if options.atom: - out.write('=', autoline=False) - if options.atom or options.cpv: - out.write(pkgs[0].key, ':', autoline=False) - out.write(stringify_attr(options, pkgs[-1], options.one_attr)) - else: - out.autoline = False - out.write(pkgs[0].key) - for attr in options.attr: - out.write(' %s="%s"' % (attr, stringify_attr(options, pkgs[-1], - attr))) - out.autoline = True - out.write() - - -def main(options, out, err): - """Run a query.""" - if options.debug: - for repo in options.repos: - out.write('repo: %r' % (repo,)) - out.write('restrict: %r' % (options.restrict,)) - out.write() - - for repo in options.repos: - try: - for pkgs in pkgutils.groupby_pkg( - repo.itermatch(options.restrict, sorter=sorted)): - pkgs = list(pkgs) - if options.noversion: - print_packages_noversion(options, out, err, pkgs) - elif options.min or options.max: - if options.min: - print_package(options, out, err, min(pkgs)) - if options.max: - print_package(options, out, err, max(pkgs)) - else: - for pkg in pkgs: - print_package(options, out, err, pkg) - if options.earlyout: - break - if options.earlyout: - break - - except KeyboardInterrupt: - raise - except Exception: - err.write('caught an exception!') - err.write('repo: %r' % (repo,)) - err.write('restrict: %r' % (options.restrict,)) - raise diff --git a/pkgcore/spawn.py b/pkgcore/spawn.py deleted file mode 100644 index 7eb8a0b..0000000 --- a/pkgcore/spawn.py +++ /dev/null @@ -1,532 +0,0 @@ -# Copyright: 2005-2006 Jason Stubbs <jstubbs@gmail.com> -# Copyright: 2004-2006 Brian Harring <ferringb@gmail.com> -# Copyright: 2004-2005 Gentoo Foundation -# License: GPL2 - - -""" -subprocess related functionality -""" - -__all__ = [ - "cleanup_pids", "spawn", "spawn_sandbox", "spawn_bash", "spawn_fakeroot", - "spawn_get_output", "find_binary"] - -import os, atexit, signal, sys - -from pkgcore.const import ( - BASH_BINARY, SANDBOX_BINARY, FAKED_PATH, LIBFAKEROOT_PATH) - -from snakeoil.osutils import listdir -from snakeoil.mappings import ProtectedDict - - -try: - import resource - max_fd_limit = resource.getrlimit(resource.RLIMIT_NOFILE)[0] -except ImportError: - max_fd_limit = 256 - -def slow_get_open_fds(): - return xrange(max_fd_limit) -if os.path.isdir("/proc/%i/fd" % os.getpid()): - def get_open_fds(): - try: - return map(int, listdir("/proc/%i/fd" % os.getpid())) - except (OSError, IOError): - return slow_get_open_fds() - except ValueError, v: - import warnings - warnings.warn( - "extremely odd, got a value error '%s' while scanning " - "/proc/%i/fd; OS allowing string names in fd?" % - (v, os.getpid())) - return slow_get_open_fds() -else: - get_open_fds = slow_get_open_fds - - -def spawn_bash(mycommand, debug=False, opt_name=None, **keywords): - """spawn the command via bash -c""" - - args = [BASH_BINARY] - if not opt_name: - opt_name = os.path.basename(mycommand.split()[0]) - if debug: - # Print commands and their arguments as they are executed. - args.append("-x") - args.append("-c") - args.append(mycommand) - return spawn(args, opt_name=opt_name, **keywords) - -def spawn_sandbox(mycommand, opt_name=None, **keywords): - """spawn the command under sandboxed""" - - if not is_sandbox_capable(): - return spawn_bash(mycommand, opt_name=opt_name, **keywords) - args = [SANDBOX_BINARY] - if not opt_name: - opt_name = os.path.basename(mycommand.split()[0]) - args.append(mycommand) - return spawn(args, opt_name=opt_name, **keywords) - -_exithandlers = [] -def atexit_register(func, *args, **kargs): - """Wrapper around atexit.register that is needed in order to track - what is registered. For example, when portage restarts itself via - os.execv, the atexit module does not work so we have to do it - manually by calling the run_exitfuncs() function in this module.""" - _exithandlers.append((func, args, kargs)) - -def run_exitfuncs(): - """This should behave identically to the routine performed by - the atexit module at exit time. It's only necessary to call this - function when atexit will not work (because of os.execv, for - example).""" - - # This function is a copy of the private atexit._run_exitfuncs() - # from the python 2.4.2 sources. The only difference from the - # original function is in the output to stderr. - exc_info = None - while _exithandlers: - func, targs, kargs = _exithandlers.pop() - try: - func(*targs, **kargs) - except SystemExit: - exc_info = sys.exc_info() - except: - exc_info = sys.exc_info() - - if exc_info is not None: - raise exc_info[0], exc_info[1], exc_info[2] - -atexit.register(run_exitfuncs) - -# We need to make sure that any processes spawned are killed off when -# we exit. spawn() takes care of adding and removing pids to this list -# as it creates and cleans up processes. -spawned_pids = [] -def cleanup_pids(pids=None): - """reap list of pids if specified, else all children""" - - if pids is None: - pids = spawned_pids - elif pids is not spawned_pids: - pids = list(pids) - - while pids: - pid = pids.pop() - try: - if os.waitpid(pid, os.WNOHANG) == (0, 0): - os.kill(pid, signal.SIGTERM) - os.waitpid(pid, 0) - except OSError: - # This pid has been cleaned up outside - # of spawn(). - pass - - if spawned_pids is not pids: - try: - spawned_pids.remove(pid) - except ValueError: - pass - -def spawn(mycommand, env=None, opt_name=None, fd_pipes=None, returnpid=False, - uid=None, gid=None, groups=None, umask=None, logfile=None, - chdir=None, path_lookup=True): - - """wrapper around execve - - @type mycommand: list or string - @type env: mapping with string keys and values - @param opt_name: controls what the process is named - (what it would show up as under top for example) - @type fd_pipes: mapping from existing fd to fd (inside the new process) - @param fd_pipes: controls what fd's are left open in the spawned process- - @param returnpid: controls whether spawn waits for the process to finish, - or returns the pid. - """ - if env is None: - env = {} - # mycommand is either a str or a list - if isinstance(mycommand, str): - mycommand = mycommand.split() - - # If an absolute path to an executable file isn't given - # search for it unless we've been told not to. - binary = mycommand[0] - if not path_lookup: - if find_binary(binary) != binary: - raise CommandNotFound(binary) - else: - binary = find_binary(binary) - - # If we haven't been told what file descriptors to use - # default to propogating our stdin, stdout and stderr. - if fd_pipes is None: - fd_pipes = {0:0, 1:1, 2:2} - - # mypids will hold the pids of all processes created. - mypids = [] - - if logfile: - # Using a log file requires that stdout and stderr - # are assigned to the process we're running. - if 1 not in fd_pipes or 2 not in fd_pipes: - raise ValueError(fd_pipes) - - # Create a pipe - (pr, pw) = os.pipe() - - # Create a tee process, giving it our stdout and stderr - # as well as the read end of the pipe. - mypids.extend(spawn(('tee', '-i', '-a', logfile), returnpid=True, - fd_pipes={0:pr, 1:fd_pipes[1], 2:fd_pipes[2]})) - - # We don't need the read end of the pipe, so close it. - os.close(pr) - - # Assign the write end of the pipe to our stdout and stderr. - fd_pipes[1] = pw - fd_pipes[2] = pw - - - pid = os.fork() - - if not pid: - # 'Catch "Exception"' - # pylint: disable-msg=W0703 - try: - _exec(binary, mycommand, opt_name, fd_pipes, env, gid, groups, - uid, umask, chdir) - except Exception, e: - # We need to catch _any_ exception so that it doesn't - # propogate out of this function and cause exiting - # with anything other than os._exit() - sys.stderr.write("%s:\n %s\n" % (e, " ".join(mycommand))) - os._exit(1) - - # Add the pid to our local and the global pid lists. - mypids.append(pid) - spawned_pids.append(pid) - - # If we started a tee process the write side of the pipe is no - # longer needed, so close it. - if logfile: - os.close(pw) - - # If the caller wants to handle cleaning up the processes, we tell - # it about all processes that were created. - if returnpid: - return mypids - - try: - # Otherwise we clean them up. - while mypids: - - # Pull the last reader in the pipe chain. If all processes - # in the pipe are well behaved, it will die when the process - # it is reading from dies. - pid = mypids.pop(0) - - # and wait for it. - retval = os.waitpid(pid, 0)[1] - - # When it's done, we can remove it from the - # global pid list as well. - spawned_pids.remove(pid) - - if retval: - # If it failed, kill off anything else that - # isn't dead yet. - for pid in mypids: - if os.waitpid(pid, os.WNOHANG) == (0, 0): - os.kill(pid, signal.SIGTERM) - os.waitpid(pid, 0) - spawned_pids.remove(pid) - - return process_exit_code(retval) - finally: - cleanup_pids(mypids) - - # Everything succeeded - return 0 - -def _exec(binary, mycommand, opt_name, fd_pipes, env, gid, groups, uid, umask, - chdir): - """internal function to handle exec'ing the child process. - - If it succeeds this function does not return. It might raise an - exception, and since this runs after fork calling code needs to - make sure this is caught and os._exit is called if it does (or - atexit handlers run twice). - """ - - # If the process we're creating hasn't been given a name - # assign it the name of the executable. - if not opt_name: - opt_name = os.path.basename(binary) - - # Set up the command's argument list. - myargs = [opt_name] - myargs.extend(mycommand[1:]) - - # Set up the command's pipes. - my_fds = {} - # To protect from cases where direct assignment could - # clobber needed fds ({1:2, 2:1}) we first dupe the fds - # into unused fds. - - protected_fds = set(fd_pipes.itervalues()) - - for trg_fd, src_fd in fd_pipes.iteritems(): - # if it's not the same we care - if trg_fd != src_fd: - if trg_fd not in protected_fds: - # if nothing we care about is there... do it now. - # we're not updating protected_fds here due to the fact - # dup will not overwrite existing fds, and that the target is - # not stated as a src at this point. - os.dup2(src_fd, trg_fd) - else: - x = os.dup(src_fd) - protected_fds.add(x) - my_fds[trg_fd] = x - - # reassign whats required now. - for trg_fd, src_fd in my_fds.iteritems(): - os.dup2(src_fd, trg_fd) - - # Then close _all_ fds that haven't been explictly - # requested to be kept open. - for fd in get_open_fds(): - # if it's not a target fd, close it. - if fd not in fd_pipes: - try: - os.close(fd) - except OSError: - pass - - if chdir is not None: - os.chdir(chdir) - - # Set requested process permissions. - if gid: - os.setgid(gid) - if groups: - os.setgroups(groups) - if uid: - os.setuid(uid) - if umask: - os.umask(umask) - - # And switch to the new process. - os.execve(binary, myargs, env) - - -def find_binary(binary, paths=None): - """look through the PATH environment, finding the binary to execute""" - - if os.path.isabs(binary): - if not (os.path.isfile(binary) and os.access(binary, os.X_OK)): - raise CommandNotFound(binary) - return binary - - if paths is None: - paths = os.environ.get("PATH", "").split(":") - - for path in paths: - filename = "%s/%s" % (path, binary) - if os.access(filename, os.X_OK) and os.path.isfile(filename): - return filename - - raise CommandNotFound(binary) - -def spawn_fakeroot(mycommand, save_file, env=None, opt_name=None, - returnpid=False, **keywords): - """spawn a process via fakeroot - - refer to the fakeroot manpage for specifics of using fakeroot - """ - if env is None: - env = {} - else: - env = ProtectedDict(env) - - if opt_name is None: - opt_name = "fakeroot %s" % mycommand - - args = [ - FAKED_PATH, - "--unknown-is-real", "--foreground", "--save-file", save_file] - - rd_fd, wr_fd = os.pipe() - daemon_fd_pipes = {1:wr_fd, 2:wr_fd} - if os.path.exists(save_file): - args.append("--load") - daemon_fd_pipes[0] = os.open(save_file, os.O_RDONLY) - else: - daemon_fd_pipes[0] = os.open("/dev/null", os.O_RDONLY) - - pids = None - pids = spawn(args, fd_pipes=daemon_fd_pipes, returnpid=True) - try: - try: - rd_f = os.fdopen(rd_fd) - line = rd_f.readline() - rd_f.close() - rd_fd = None - except: - cleanup_pids(pids) - raise - finally: - for x in (rd_fd, wr_fd, daemon_fd_pipes[0]): - if x is not None: - try: - os.close(x) - except OSError: - pass - - line = line.strip() - - try: - fakekey, fakepid = map(int, line.split(":")) - except ValueError: - raise ExecutionFailure("output from faked was unparsable- %s" % line) - - # by now we have our very own daemonized faked. yay. - env["FAKEROOTKEY"] = str(fakekey) - env["LD_PRELOAD"] = ":".join( - [LIBFAKEROOT_PATH] + env.get("LD_PRELOAD", "").split(":")) - - try: - ret = spawn( - mycommand, opt_name=opt_name, env=env, returnpid=returnpid, - **keywords) - if returnpid: - return ret + [fakepid] + pids - return ret - finally: - if not returnpid: - cleanup_pids([fakepid] + pids) - -def spawn_get_output( - mycommand, spawn_type=spawn, raw_exit_code=False, collect_fds=(1,), - fd_pipes=None, split_lines=True, **keywords): - - """Call spawn, collecting the output to fd's specified in collect_fds list. - - @param spawn_type: the passed in function to call- - typically spawn_bash, spawn, spawn_sandbox, or spawn_fakeroot. - defaults to spawn - """ - - pr, pw = None, None - if fd_pipes is None: - fd_pipes = {0:0} - else: - fd_pipes = ProtectedDict(fd_pipes) - try: - pr, pw = os.pipe() - for x in collect_fds: - fd_pipes[x] = pw - keywords["returnpid"] = True - mypid = spawn_type(mycommand, fd_pipes=fd_pipes, **keywords) - os.close(pw) - pw = None - - if not isinstance(mypid, (list, tuple)): - raise ExecutionFailure() - - fd = os.fdopen(pr, "r") - try: - if not split_lines: - mydata = fd.read() - else: - mydata = fd.readlines() - finally: - fd.close() - pw = None - - retval = os.waitpid(mypid[0], 0)[1] - cleanup_pids(mypid) - if raw_exit_code: - return [retval, mydata] - return [process_exit_code(retval), mydata] - - finally: - if pr is not None: - try: - os.close(pr) - except OSError: - pass - if pw is not None: - try: - os.close(pw) - except OSError: - pass - -def process_exit_code(retval): - """Process a waitpid returned exit code. - - @return: The exit code if it exit'd, the signal if it died from signalling. - """ - # If it got a signal, return the signal that was sent. - if retval & 0xff: - return (retval & 0xff) << 8 - - # Otherwise, return its exit code. - return retval >> 8 - - -class ExecutionFailure(Exception): - def __init__(self, msg): - Exception.__init__(self, msg) - self.msg = msg - def __str__(self): - return "Execution Failure: %s" % self.msg - -class CommandNotFound(ExecutionFailure): - def __init__(self, command): - ExecutionFailure.__init__( - self, "CommandNotFound Exception: Couldn't find '%s'" % (command,)) - self.command = command - -# cached capabilities - -def is_fakeroot_capable(force=False): - if not force: - try: - return is_fakeroot_capable.cached_result - except AttributeError: - pass - if not (os.path.exists(FAKED_PATH) and os.path.exists(LIBFAKEROOT_PATH)): - res = False - else: - try: - r, s = spawn_get_output(["fakeroot", "--version"], - fd_pipes={2:1, 1:1}) - res = (r == 0) and (len(s) == 1) and ("version 1." in s[0]) - except ExecutionFailure: - res = False - is_fakeroot_capable.cached_result = res - return res - -def is_sandbox_capable(force=False): - if not force: - try: - return is_sandbox_capable.cached_result - except AttributeError: - pass - res = os.path.isfile(SANDBOX_BINARY) and os.access(SANDBOX_BINARY, os.X_OK) - is_sandbox_capable.cached_result = res - return res - -def is_userpriv_capable(force=False): - if not force: - try: - return is_userpriv_capable.cached_result - except AttributeError: - pass - res = is_userpriv_capable.cached_result = (os.getuid() == 0) - return res diff --git a/pkgcore/sync/__init__.py b/pkgcore/sync/__init__.py deleted file mode 100644 index bd8f25d..0000000 --- a/pkgcore/sync/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# Copyright: 2006 Brian Harring <ferringb@gmail.com> -# License: GPL2 - diff --git a/pkgcore/sync/base.py b/pkgcore/sync/base.py deleted file mode 100644 index 8ebf046..0000000 --- a/pkgcore/sync/base.py +++ /dev/null @@ -1,175 +0,0 @@ -# Copyright: 2006 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -from pkgcore.config import ConfigHint, configurable -from snakeoil import demandload, descriptors -demandload.demandload(globals(), - 'os', - 'pwd', - 'stat', - 'errno', - 'pkgcore:spawn', - 'pkgcore:plugin', - 'pkgcore:os_data', -) - - -class syncer_exception(Exception): - pass - -class uri_exception(syncer_exception): - pass - -class generic_exception(syncer_exception): - pass - -class missing_local_user(syncer_exception): - pass - -class missing_binary(syncer_exception): - pass - - -class syncer(object): - - forcable = False - - supported_uris = () - - pkgcore_config_type = ConfigHint( - {'path':'str', 'uri':'str'}, typename='syncer') - - def __init__(self, path, uri, default_verbosity=0): - self.verbose = default_verbosity - self.basedir = path.rstrip(os.path.sep) + os.path.sep - self.local_user, self.uri = self.split_users(uri) - - @staticmethod - def split_users(raw_uri): - """ - @param raw_uri: string uri to split users from; harring::ferringb:pass - for example is local user 'harring', remote 'ferringb', - password 'pass' - @return: (local user, remote user, remote pass), defaults to root_uid - if no local user specified - """ - uri = raw_uri.split("::", 1) - if len(uri) == 1: - return os_data.root_uid, raw_uri - try: - if uri[1].startswith("@"): - uri[1] = uri[1][1:] - if '/' in uri[0] or ':' in uri[0]: - proto = uri[0].split("/", 1) - proto[1] = proto[1].lstrip("/") - uri[0] = proto[1] - uri[1] = "%s//%s" % (proto[0], uri[1]) - return pwd.getpwnam(uri[0]).pw_uid, uri[1] - except KeyError, e: - raise missing_local_user(raw_uri, uri[0], e) - - def sync(self, verbosity=None, force=False): - kwds = {} - if self.forcable and force: - kwds["force"] = True - if verbosity is None: - verbosity = self.verbose - # output_fd is harded coded as stdout atm. - return self._sync(verbosity, 1, **kwds) - - def _sync(self, verbosity, output_fd, **kwds): - raise NotImplementedError(self, "_sync") - - def __str__(self): - return "%s syncer: %s, %s" % (self.__class__, - self.basedir, self.uri) - - @classmethod - def supports_uri(cls, uri): - for prefix, level in cls.supported_uris: - if uri.startswith(prefix): - return level - return 0 - - -class ExternalSyncer(syncer): - - """Base class for syncers that spawn a binary to do the the actual work.""" - - sets_env = False - binary = None - - def __init__(self, path, uri, default_verbosity=0): - syncer.__init__(self, path, uri, default_verbosity=default_verbosity) - if not self.sets_env: - self.env = {} - if not hasattr(self, 'binary_path'): - self.binary_path = self.require_binary(self.binary) - - @staticmethod - def require_binary(bin_name, fatal=True): - try: - return spawn.find_binary(bin_name) - except spawn.CommandNotFound, e: - if fatal: - raise missing_binary(bin_name, e) - return None - - @descriptors.classproperty - def disabled(cls): - disabled = getattr(cls, '_disabled', None) - if disabled is None: - path = getattr(cls, 'binary_path', None) - if path is None: - disabled = cls._disabled = ( - cls.require_binary(cls.binary, fatal=False) is None) - else: - disabled = cls._disabled = os.path.exists(path) - return disabled - - def set_binary_path(self): - self.binary_path = self.require_binary(self.binary) - - def _spawn(self, command, pipes, **kwargs): - return spawn.spawn(command, fd_pipes=pipes, uid=self.local_user, - env=self.env, **kwargs) - - -class dvcs_syncer(ExternalSyncer): - - def _sync(self, verbosity, output_fd): - try: - st = os.stat(self.basedir) - except (IOError, OSError), ie: - if ie.errno != errno.ENOENT: - raise generic_exception(self, self.basedir, ie) - command = self._initial_pull() - chdir = None - else: - if not stat.S_ISDIR(st.st_mode): - raise generic_exception(self, self.basedir, - "isn't a directory") - command = self._update_existing() - chdir = self.basedir - - ret = self._spawn(command, {1:output_fd, 2:output_fd, 0:0}, - chdir=chdir) - return ret == 0 - - def _initial_pull(self): - raise NotImplementedError(self, "_initial_pull") - - def _update_existing(self): - raise NotImplementedError(self, "_update_existing") - -@configurable({'basedir':'str', 'uri':'str'}, typename='syncer') -def GenericSyncer(basedir, uri, default_verbosity=0): - """Syncer using the plugin system to find a syncer based on uri.""" - plugins = list( - (plug.supports_uri(uri), plug) - for plug in plugin.get_plugins('syncer')) - plugins.sort() - if not plugins or plugins[-1][0] <= 0: - raise uri_exception('no known syncer supports %r' % (uri,)) - # XXX this is random if there is a tie. Should we raise an exception? - return plugins[-1][1](basedir, uri, default_verbosity=default_verbosity) diff --git a/pkgcore/sync/bzr.py b/pkgcore/sync/bzr.py deleted file mode 100644 index ee5fd8b..0000000 --- a/pkgcore/sync/bzr.py +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright: 2006 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -from pkgcore.sync import base - -class bzr_syncer(base.dvcs_syncer): - - binary = "bzr" - - supported_uris = ( - ('bzr+', 5), - ) - - @staticmethod - def parse_uri(raw_uri): - if not raw_uri.startswith("bzr+"): - raise base.uri_exception(raw_uri, "doesn't start with bzr+") - return raw_uri[4:] - - def __init__(self, basedir, uri, **kwargs): - uri = self.parse_uri(uri) - base.dvcs_syncer.__init__(self, basedir, uri, **kwargs) - - def _initial_pull(self): - return [self.binary_path, "get", self.basedir, self.uri] - - def _update_existing(self): - return [self.binary_path, "pull", self.uri] diff --git a/pkgcore/sync/cvs.py b/pkgcore/sync/cvs.py deleted file mode 100644 index 14e351d..0000000 --- a/pkgcore/sync/cvs.py +++ /dev/null @@ -1,57 +0,0 @@ -# Copyright: 2006 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -from pkgcore.sync import base - -class cvs_syncer(base.dvcs_syncer): - - sets_env = True - binary = "cvs" - - supported_uris = ( - ('cvs+', 5), - ('cvs://', 5), - ) - - @classmethod - def parse_uri(cls, raw_uri): - if not raw_uri.startswith("cvs") and \ - not raw_uri.startswith("cvs+"): - raise base.uri_exception(raw_uri, "must be cvs:// or cvs+${RSH}") - if raw_uri.startswith("cvs://"): - return None, raw_uri[len("cvs://"):] - proto = raw_uri[len("cvs+"):].split(":", 1) - if not proto[0]: - raise base.uri_exception(raw_uri, - "cvs+ requires the rsh alternative to be specified") - if proto[0] == "anon": - proto[0] = None - elif proto[0] != "pserver": - proto[0] = cls.require_binary(proto[0]) - return proto[0], proto[1].lstrip("/") - - def __init__(self, basedir, raw_uri, **kwargs): - proto, uri = self.parse_uri(raw_uri) - self.rsh = proto - if self.rsh is None: - uri = ":anoncvs:%s" % uri - elif self.rsh == "pserver": - uri = ":pserver:%s" % uri - self.rsh = None - else: - uri = ":ext:%s" % uri - host, self.module = uri.rsplit(":", 1) - base.dvcs_syncer.__init__(self, basedir, host, **kwargs) - - @property - def env(self): - k = {"CVSROOT":self.uri} - if self.rsh is not None: - k["CVS_RSH"] = self.rsh - return k - - def _update_existing(self): - return [self.binary_path, "up"] - - def _initial_pull(self): - return [self.binary_path, "co", "-d", self.basedir] diff --git a/pkgcore/sync/darcs.py b/pkgcore/sync/darcs.py deleted file mode 100644 index 3141a4e..0000000 --- a/pkgcore/sync/darcs.py +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright: 2006 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -from pkgcore.sync import base - -class darcs_syncer(base.dvcs_syncer): - - binary = "darcs" - - supported_uris = ( - ('darcs+', 5), - ) - - @staticmethod - def parse_uri(raw_uri): - if not raw_uri.startswith("darcs+"): - raise base.uri_exception(raw_uri, "doesn't start with darcs+") - return raw_uri[6:] - - def __init__(self, basedir, uri, **kwargs): - uri = self.parse_uri(uri) - base.dvcs_syncer.__init__(self, basedir, uri, **kwargs) - - def _initial_pull(self): - return [self.binary_path, "clone", self.uri, self.basedir] - - def _update_existing(self): - return [self.binary_path, "pull", self.uri] diff --git a/pkgcore/sync/git.py b/pkgcore/sync/git.py deleted file mode 100644 index 10bf4d6..0000000 --- a/pkgcore/sync/git.py +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright: 2006 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -from pkgcore.sync import base - -class git_syncer(base.dvcs_syncer): - - binary = "git" - - supported_uris = ( - ('git://', 5), - ('git+', 5), - ) - - @staticmethod - def parse_uri(raw_uri): - if not raw_uri.startswith("git+") and not raw_uri.startswith("git://"): - raise base.uri_exception(raw_uri, - "doesn't start with git+ nor git://") - if raw_uri.startswith("git+"): - if raw_uri.startswith("git+:"): - raise base.uri_exception(raw_uri, - "need to specify the sub protocol if using git+") - return raw_uri[4:] - return raw_uri - - def __init__(self, basedir, uri, **kwargs): - uri = self.parse_uri(uri) - base.dvcs_syncer.__init__(self, basedir, uri, **kwargs) - - def _initial_pull(self): - return [self.binary_path, "clone", self.uri, self.basedir] - - def _update_existing(self): - return [self.binary_path, "pull"] diff --git a/pkgcore/sync/hg.py b/pkgcore/sync/hg.py deleted file mode 100644 index 3ab3c78..0000000 --- a/pkgcore/sync/hg.py +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright: 2006 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -from pkgcore.sync import base - -class hg_syncer(base.dvcs_syncer): - - binary = "hg" - - supported_uris = ( - ('hg+', 5), - ) - - @staticmethod - def parse_uri(raw_uri): - if not raw_uri.startswith("hg+"): - raise base.uri_exception(raw_uri, "doesn't start with hg+") - return raw_uri[3:] - - def __init__(self, basedir, uri, **kwargs): - uri = self.parse_uri(uri) - base.dvcs_syncer.__init__(self, basedir, uri, **kwargs) - - def _initial_pull(self): - return [self.binary_path, "clone", self.uri, self.basedir] - - def _update_existing(self): - return [self.binary_path, "pull", "-u", self.uri] diff --git a/pkgcore/sync/rsync.py b/pkgcore/sync/rsync.py deleted file mode 100644 index e46d257..0000000 --- a/pkgcore/sync/rsync.py +++ /dev/null @@ -1,188 +0,0 @@ -# Copyright: 2006 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -from pkgcore.sync import base -from pkgcore.config import ConfigHint -from snakeoil.demandload import demandload - -demandload(globals(), - 'os', - 'socket', - 'errno', - 'snakeoil.osutils:pjoin', -) - -class rsync_syncer(base.ExternalSyncer): - - default_excludes = ["/distfiles", "/local", "/packages"] - default_includes = [] - default_timeout = 180 - default_opts = ["--recursive", - "--delete-after", - "--perms", - "--times", - "--force", - "--safe-links", - "--whole-file"] # this one probably shouldn't be a default. - - default_retries = 5 - binary = "rsync" - - @classmethod - def parse_uri(cls, raw_uri): - if not raw_uri.startswith("rsync://") and \ - not raw_uri.startswith("rsync+"): - raise base.uri_exception(raw_uri, - "doesn't start with rsync:// nor rsync+") - - if raw_uri.startswith("rsync://"): - return None, raw_uri - - proto = raw_uri.split(":", 1) - proto[0] = proto[0].split("+", 1)[1] - cls.require_binary(proto[0]) - return proto[0], "rsync:%s" % proto[1] - - pkgcore_config_type = ConfigHint({'basedir':'str', 'uri':'str', - 'timeout':'str', 'compress':'bool', 'excludes':'list', - 'includes':'list', 'retries':'str', 'extra_opts':'list'}, - typename='syncer') - - def __init__(self, basedir, uri, timeout=default_timeout, - compress=False, excludes=(), includes=(), - retries=default_retries, - extra_opts=[]): - - uri = uri.rstrip(os.path.sep) + os.path.sep - self.rsh, uri = self.parse_uri(uri) - base.ExternalSyncer.__init__(self, basedir, uri, default_verbosity=2) - self.hostname = self.parse_hostname(self.uri) - if self.rsh: - self.rsh = self.require_binary(self.rsh) - self.opts = list(self.default_opts) - self.opts.extend(extra_opts) - if compress: - self.opts.append("--compress") - self.opts.append("--timeout=%i" % int(timeout)) - self.excludes = list(self.default_excludes) + list(excludes) - self.includes = list(self.default_includes) + list(includes) - self.retries = int(retries) - self.is_ipv6 = "--ipv6" in self.opts or "-6" in self.opts - self.is_ipv6 = self.is_ipv6 and socket.has_ipv6 - - @staticmethod - def parse_hostname(uri): - return uri[len("rsync://"):].split("@", 1)[-1].split("/", 1)[0] - - def _get_ips(self): - af_fam = socket.AF_INET - if self.is_ipv6: - af_fam = socket.AF_INET6 - try: - for ipaddr in socket.getaddrinfo(self.hostname, None, af_fam, - socket.SOCK_STREAM): - if ipaddr[0] == socket.AF_INET6: - yield "[%s]" % ipaddr[4][0] - else: - yield ipaddr[4][0] - - except socket.error, e: - raise base.syncer_exception(self.hostname, af_fam, str(e)) - - - def _sync(self, verbosity, output_fd): - fd_pipes = {1:output_fd, 2:output_fd} - opts = list(self.opts) - if self.rsh: - opts.append("-e") - opts.append(self.rsh) - opts.extend("--exclude=%s" % x for x in self.excludes) - opts.extend("--include=%s" % x for x in self.includes) - if verbosity == 0: - opts.append("--quiet") - if verbosity >= 1: - opts.append("--progress") - if verbosity >= 2: - opts.append("--stats") - elif verbosity >= 3: - opts.append("--verbose") - - # zip limits to the shortest iterable. - for count, ip in zip(xrange(self.retries), self._get_ips()): - o = [self.binary_path, - self.uri.replace(self.hostname, ip, 1), - self.basedir] + opts - - ret = self._spawn(o, fd_pipes) - if ret == 0: - return True - elif ret == 1: - # syntax error. - raise base.syncer_exception(o, "syntax error") - elif ret == 11: - raise base.syncer_exception("rsync returned error code of " - "11; this is an out of space exit code") - # need to do something here instead of just restarting... - # else: - # print ret - - -class rsync_timestamp_syncer(rsync_syncer): - - forcable = True - - def __init__(self, *args, **kwargs): - rsync_syncer.__init__(self, *args, **kwargs) - self.last_timestamp = self.current_timestamp() - - def current_timestamp(self, path=None): - """ - @param path: override the default path for the timestamp to read - @return: string of the timestamp data - """ - if path is None: - path = pjoin(self.basedir, "metadata", "timestamp.chk") - try: - return open(path).read().strip() - except IOError, oe: - if oe.errno not in (errno.ENOENT, errno.ENOTDIR): - raise - return None - - def _sync(self, verbosity, output_fd, force=False): - doit = force or self.last_timestamp is None - ret = None - try: - if not doit: - basedir = self.basedir - uri = self.uri - new_timestamp = pjoin(self.basedir, "metadata", - ".tmp.timestamp.chk") - try: - self.basedir = new_timestamp - self.uri = pjoin(self.uri, "metadata", "timestamp.chk") - ret = rsync_syncer._sync(self, verbosity, output_fd) - finally: - self.basedir = basedir - self.uri = uri - doit = ret == False or self.last_timestamp != \ - self.current_timestamp(new_timestamp) - if not doit: - return True - ret = rsync_syncer._sync(self, verbosity, output_fd) - finally: - if ret is not None: - if ret: - return ret - # ensure the timestamp is back to the old. - try: - path = pjoin(self.basedir, "metadata", "timestamp.chk") - if self.last_timestamp is None: - os.remove(path) - else: - open(pjoin(self.basedir, "metadata", "timestamp.chk"), - "w").write(self.last_timestamp) - except (IOError, OSError): - # don't care... - pass - return ret diff --git a/pkgcore/sync/svn.py b/pkgcore/sync/svn.py deleted file mode 100644 index cefeace..0000000 --- a/pkgcore/sync/svn.py +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright: 2006 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -from pkgcore.sync import base - -class svn_syncer(base.ExternalSyncer): - - binary = "svn" - - supported_uris = ( - ('svn://', 5), - ('svn+', 5), - ('http+svn://',5), - ('https+svn://',5) - ) - - @staticmethod - def parse_uri(raw_uri): - if raw_uri.startswith("svn://"): - return True - elif raw_uri.startswith("http+svn://"): - return True - elif raw_uri.startswith("https+svn://"): - return True - elif raw_uri.startswith("svn+"): - if raw_uri.startswith("svn+:"): - raise base.uri_exception(raw_uri, "svn+:// isn't valid") - else: - raise base.uri_exception(raw_uri, "protocol unknown") - return True - - def _sync(self, verbosity, output_fd): - uri = self.uri - if uri.startswith('svn+http://'): - uri = uri.replace('svn+http://', 'http://') - elif uri.startswith('svn+https://'): - uri = uri.replace('svn+https://', 'https://') - return 0 == self._spawn([self.binary_path, "co", - uri, self.basedir], {1:output_fd, 2:output_fd, 0:0}) - diff --git a/pkgcore/util/__init__.py b/pkgcore/util/__init__.py deleted file mode 100644 index 9ff5a09..0000000 --- a/pkgcore/util/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -# Copyright: 2005 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -"""misc. utility functions""" diff --git a/pkgcore/util/bzip2.py b/pkgcore/util/bzip2.py deleted file mode 100644 index afbcb39..0000000 --- a/pkgcore/util/bzip2.py +++ /dev/null @@ -1,69 +0,0 @@ -# Copyright: 2006 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -""" -bzip2 decompression/compression - -where possible, this module defers to cpython bz2 module- if it's not available, -it results to executing bzip2 with tempfile arguements to do decompression -and compression. - -Should use this module unless its absolutely critical that bz2 module be used -""" - - -from snakeoil.demandload import demandload -demandload(globals(), - 'tempfile', - 'pkgcore.spawn:find_binary,spawn_get_output', -) - -def process_compress(in_data, compress_level=9): - fd = None - fd = tempfile.TemporaryFile("w+") - fd.write(in_data) - fd.flush() - fd.seek(0) - try: - ret, data = spawn_get_output( - ["bzip2", "-%ic" % compress_level], - fd_pipes={0:fd.fileno()}, split_lines=False) - if ret != 0: - raise ValueError("failed compressing the data") - return data - finally: - if fd is not None: - fd.close() - -def process_decompress(in_data): - fd = None - fd = tempfile.TemporaryFile("wb+") - fd.write(in_data) - fd.flush() - fd.seek(0) - try: - ret, data = spawn_get_output( - ["bzip2", "-dc"], fd_pipes={0:fd.fileno()}, split_lines=False) - if ret != 0: - raise ValueError("failed decompressing the data") - return data - finally: - if fd is not None: - fd.close() - -# Unused import -# pylint: disable-msg=W0611 - -try: - from bz2 import compress, decompress -except ImportError: - # We need this because if we are not native then TarFile.bz2open will fail - # (and some code needs to be able to check that). - native = False - # trigger it to throw a CommandNotFound if missing - find_binary("bzip2") - compress = process_compress - decompress = process_decompress -else: - native = True - diff --git a/pkgcore/util/commandline.py b/pkgcore/util/commandline.py deleted file mode 100644 index f6cd123..0000000 --- a/pkgcore/util/commandline.py +++ /dev/null @@ -1,425 +0,0 @@ -# Copyright: 2006 Marien Zwart <marienz@gentoo.org> -# License: GPL2 - - -"""Utilities for writing commandline utilities. - -pkgcore scripts should use the L{OptionParser} subclass here for a -consistent commandline "look and feel" (and it tries to make life a -bit easier too). They will probably want to use L{main} from an C{if -__name__ == '__main__'} block too: it will take care of things like -consistent exception handling. - -See dev-notes/commandline.rst for more complete documentation. -""" - - -import sys -import os.path -import logging - -from pkgcore.config import load_config, errors -from snakeoil import formatters, demandload, fix_copy -fix_copy.inject_copy() -import optparse - -demandload.demandload(globals(), - 'snakeoil.fileutils:iter_read_bash', - 'pkgcore:version', - 'pkgcore.config:basics', - 'pkgcore.restrictions:packages', - 'pkgcore.util:parserestrict', -) - - -CONFIG_LOADED_MSG = ( - 'Configuration already loaded. If moving the option earlier ' - 'on the commandline does not fix this report it as a bug.') - - -class FormattingHandler(logging.Handler): - - """Logging handler printing through a formatter.""" - - def __init__(self, formatter): - logging.Handler.__init__(self) - # "formatter" clashes with a Handler attribute. - self.out = formatter - - def emit(self, record): - if record.levelno >= logging.ERROR: - color = 'red' - elif record.levelno >= logging.WARNING: - color = 'yellow' - else: - color = 'cyan' - first_prefix = (self.out.fg(color), self.out.bold, record.levelname, - self.out.reset, ' ', record.name, ': ') - later_prefix = (len(record.levelname) + len(record.name)) * ' ' + ' : ' - self.out.first_prefix.extend(first_prefix) - self.out.later_prefix.append(later_prefix) - try: - for line in self.format(record).split('\n'): - self.out.write(line, wrap=True) - finally: - self.out.later_prefix.pop() - for i in xrange(len(first_prefix)): - self.out.first_prefix.pop() - - -# Mix in object here or properties do not work (Values is an oldstyle class). -class Values(optparse.Values, object): - - """Values with an autoloaded config property. - - If you do not want the config autoloaded you can set the _config - attribute like this: - - >>> parser = OptionParser() - >>> vals = parser.get_default_values() - >>> vals._config = my_custom_central - >>> parser.parse_args(args, vals) - """ - - def __init__(self, defaults=None): - optparse.Values.__init__(self, defaults) - self.new_config = {} - self.add_config = {} - - def load_config(self): - """Override this if you need a different way of loading config.""" - # This makes mixing --new-config and --add-config sort of - # work. Not sure if that is a good thing, but detecting and - # erroring is about as much work as making it mostly work :) - new_config = dict( - (name, basics.ConfigSectionFromStringDict(val)) - for name, val in self.new_config.iteritems()) - add_config = {} - for name, config in self.add_config.iteritems(): - inherit = config.pop('inherit', None) - # XXX this will likely not be quite correctly quoted. - if inherit is None: - config['inherit'] = repr(name) - else: - config['inherit'] = '%s %r' % (inherit, name) - add_config[name] = basics.ConfigSectionFromStringDict(config) - # Triggers failures if these get mucked with after this point - # (instead of silently ignoring). - self.add_config = self.new_config = None - return load_config( - debug=self.debug, prepend_sources=(add_config, new_config), - skip_config_files=self.empty_config) - - @property - def config(self): - try: - return self._config - except AttributeError: - self._config = self.load_config() - return self._config - - -def read_file_callback(option, opt_str, value, parser): - """Read a file ignoring comments.""" - if not os.path.isfile(value): - raise optparse.OptionValueError("'%s' is not a file" % value) - setattr(parser.values, option.dest, iter_read_bash(value)) - - -def config_callback(option, opt_str, value, parser, typename, typedesc=None): - """Retrieve a config section. - - Pass the typename of the section as callback_args=('typename',), - and set type='string'. You can optionally pass a human-readable - typename as second element of callback_args. - """ - if typedesc is None: - typedesc = typename - mapping = getattr(parser.values.config, typename) - try: - result = mapping[value] - except KeyError: - raise optparse.OptionValueError( - '%r is not a valid %s for %s (valid values: %s)' % ( - value, typedesc, opt_str, ', '.join(repr(key) - for key in mapping))) - setattr(parser.values, option.dest, result) - - -def config_append_callback(option, opt_str, value, parser, typename, - typedesc=None): - """Like L{config_callback} but appends instead of sets.""" - if typedesc is None: - typedesc = typename - mapping = getattr(parser.values.config, typename) - try: - result = mapping[value] - except KeyError: - raise optparse.OptionValueError( - '%r is not a valid %s for %s (valid values: %s)' % ( - value, typedesc, opt_str, ', '.join(repr(key) - for key in mapping))) - parser.values.ensure_value(option.dest, []).append(result) - - -def debug_callback(option, opt_str, value, parser): - """Make sure the config central uses debug mode. - - We do this because it is possible to access config from an option - callback before the entire commandline is parsed. This callback - makes sure any config usage after optparse hit the --debug switch - is properly in debug mode. - - Ideally we would not need this, since needing this means things - accessing config too early still get the wrong debug setting. But - doing that would mean either (crappily) parsing the commandline - before optparse does or making config access from option callbacks - illegal. The former is hard to get "right" (impossible to get - completely "right" since you cannot know how many arguments an - option with a callback consumes without calling it) and the latter - is unwanted because accessing config from callbacks is useful - (pcheck will do this at the time of writing). - """ - parser.values.debug = True - config = parser.values.config - config.debug = True - logging.root.setLevel(logging.DEBUG) - for collapsed in config.collapsed_configs.itervalues(): - collapsed.debug = True - - -def new_config_callback(option, opt_str, value, parser): - """Add a configsection to our values object. - - Munges three arguments: section name, key name, value. - - dest defines an attr name on the values object to store in. - """ - if getattr(parser.values, '_config', None) is not None: - raise optparse.OptionValueError(CONFIG_LOADED_MSG) - section_name, key, val = value - section = getattr(parser.values, option.dest).setdefault(section_name, {}) - if key in section: - raise optparse.OptionValueError( - '%r is already set (to %r)' % (key, section[key])) - section[key] = val - - -def empty_config_callback(option, opt_str, value, parser): - """Remember not to load the user/system configuration. - - Error out if we have already loaded it. - """ - if getattr(parser.values, '_config', None) is not None: - raise optparse.OptionValueError(CONFIG_LOADED_MSG) - parser.values.empty_config = True - - -class Option(optparse.Option): - - def __init__(self, *args, **kwargs): - self.long_help = kwargs.pop('long_help', None) - optparse.Option.__init__(self, *args, **kwargs) - - -class OptionParser(optparse.OptionParser): - - """Our common OptionParser subclass. - - Adds some common options, makes options that get "append"ed - default to an empty sequence instead of None, uses our custom - Values class with the config property. - """ - - # You can set this on an instance or subclass to use a different class. - values_class = Values - - standard_option_list = optparse.OptionParser.standard_option_list + [ - Option( - '--debug', '-d', action='callback', callback=debug_callback, - help='print some extra info useful for pkgcore devs. You may have ' - 'to set this as first argument for debugging certain ' - 'configuration problems.'), - Option('--nocolor', action='store_true', - help='disable color in the output.'), - Option('--version', action='version'), - Option( - '--add-config', action='callback', callback=new_config_callback, - type='str', nargs=3, help='Add a new configuration section. ' - 'Takes three arguments: section name, value name, value.'), - Option( - '--new-config', action='callback', callback=new_config_callback, - type='str', nargs=3, help='Expand a configuration section. ' - 'Just like --add-config but with an implied inherit=sectionname.'), - Option( - '--empty-config', action='callback', - callback=empty_config_callback, - help='Do not load the user or system configuration. Can be useful ' - 'combined with --new-config.') - ] - - def __init__(self, *args, **kwargs): - """Initialize.""" - kwargs.setdefault('option_class', Option) - optparse.OptionParser.__init__(self, *args, **kwargs) - # It is a callback so it cannot set a default value the "normal" way. - self.set_default('debug', False) - self.set_default('empty_config', False) - - def get_version(self): - """Add pkgcore's version to the version information.""" - ver = optparse.OptionParser.get_version(self) - pkgcore_ver = version.get_version() - if ver: - return '\n'.join((ver, pkgcore_ver)) - return pkgcore_ver - - def print_version(self, file=None): - """Print the version to a filelike (defaults to stdout). - - Overridden because the optparse one is a noop if self.version is false. - """ - print >> file, self.get_version() - - def _add_version_option(self): - """Override this to be a no-op. - - Needed because optparse does not like our on-demand generation - of the version string. - """ - - def get_default_values(self): - """Slightly simplified copy of optparse code using our Values class.""" - # Needed because optparse has the Values class hardcoded in - # (and no obvious way to get the defaults set on an existing - # Values instance). - defaults = self.defaults.copy() - for option in self._get_all_options(): - default = defaults.get(option.dest) - if isinstance(default, basestring): - opt_str = option.get_opt_string() - defaults[option.dest] = option.check_value(opt_str, default) - return self.values_class(defaults) - - def check_values(self, values, args): - """Do some basic sanity checking. - - optparse defaults unset lists to None. An empty sequence is - much more convenient (lets you use them in a for loop without - a None check) so we fix those up (based on action "append"). - """ - for container in self.option_groups + [self]: - for option in container.option_list: - if option.action == 'append': - values.ensure_value(option.dest, []) - return values, args - -class MySystemExit(SystemExit): - """Subclass of SystemExit the tests can safely catch.""" - - -def main(subcommands, args=None, outfile=sys.stdout, errfile=sys.stderr, - script_name=None): - """Function to use in an "if __name__ == '__main__'" block in a script. - - Takes one or more combinations of option parser and main func and - runs them, taking care of exception handling and some other things. - - Any ConfigurationErrors raised from your function (by the config - manager) are handled. Other exceptions are not (trigger a traceback). - - @type subcommands: mapping of string => (OptionParser class, main func) - @param subcommands: available commands. - The keys are a subcommand name or None for other/unknown/no subcommand. - The values are tuples of OptionParser subclasses and functions called - as main_func(config, out, err) with a L{Values} instance, two - L{snakeoil.formatters.Formatter} instances for output (stdout) - and errors (stderr). It should return an integer used as - exit status or None as synonym for 0. - @type args: sequence of strings - @param args: arguments to parse, defaulting to C{sys.argv[1:]}. - @type outfile: file-like object - @param outfile: File to use for stdout, defaults to C{sys.stdout}. - @type errfile: file-like object - @param errfile: File to use for stderr, defaults to C{sys.stderr}. - @type script_name: string - @param script_name: basename of this script, defaults to the basename - of C{sys.argv[0]}. - """ - exitstatus = 1 - if args is None: - args = sys.argv[1:] - if script_name is None: - prog = os.path.basename(sys.argv[0]) - else: - prog = script_name - parser_class = None - if args: - parser_class, main_func = subcommands.get(args[0], (None, None)) - if parser_class is not None: - prog = '%s %s' % (prog, args[0]) - args = args[1:] - if parser_class is None: - try: - parser_class, main_func = subcommands[None] - except KeyError: - # This tries to print in a format very similar to optparse --help. - errfile.write( - 'Usage: %s <command>\n\nCommands:\n' % (prog,)) - maxlen = max(len(subcommand) for subcommand in subcommands) + 1 - for subcommand, (parser, main) in sorted(subcommands.iteritems()): - doc = main.__doc__ - if doc is None: - errfile.write(' %s\n' % (subcommand,)) - else: - doc = doc.split('\n', 1)[0] - errfile.write(' %-*s %s\n' % (maxlen, subcommand, doc)) - errfile.write( - '\nUse --help after a subcommand for more help.\n') - raise MySystemExit(1) - options = None - option_parser = parser_class(prog=prog) - out = None - try: - options, args = option_parser.parse_args(args) - # Checked here and not in OptionParser because we want our - # check_values to run before the user's, not after it. - if args: - option_parser.error("I don't know what to do with %s" % - (' '.join(args),)) - else: - if options.nocolor: - formatter_factory = formatters.PlainTextFormatter - else: - formatter_factory = formatters.get_formatter - out = formatter_factory(outfile) - err = formatter_factory(errfile) - if logging.root.handlers: - # Remove the default handler. - logging.root.handlers.pop(0) - logging.root.addHandler(FormattingHandler(err)) - exitstatus = main_func(options, out, err) - except errors.ConfigurationError, e: - if options is not None and options.debug: - raise - errfile.write('Error in configuration:\n%s\n' % (e,)) - except KeyboardInterrupt: - if options is not None and options.debug: - raise - if out is not None: - if exitstatus: - out.title('%s failed' % (prog,)) - else: - out.title('%s succeeded' % (prog,)) - raise MySystemExit(exitstatus) - -def convert_to_restrict(sequence, default=packages.AlwaysTrue): - """Convert an iterable to a list of atoms, or return the default""" - l = [] - try: - for x in sequence: - l.append(parserestrict.parse_match(x)) - except parserestrict.ParseError, e: - raise optparse.OptionValueError("arg %r isn't a valid atom: %s" - % (x, e)) - return l or [default] diff --git a/pkgcore/util/packages.py b/pkgcore/util/packages.py deleted file mode 100644 index bbdd649..0000000 --- a/pkgcore/util/packages.py +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright: 2006 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -import itertools, operator - -def get_raw_pkg(pkg): - p = pkg - while hasattr(p, "_raw_pkg"): - p = p._raw_pkg - return p - -groupby_key_getter = operator.attrgetter("key") -def groupby_pkg(iterable): - for key, pkgs in itertools.groupby(iterable, groupby_key_getter): - yield pkgs diff --git a/pkgcore/util/parserestrict.py b/pkgcore/util/parserestrict.py deleted file mode 100644 index 4f7f17c..0000000 --- a/pkgcore/util/parserestrict.py +++ /dev/null @@ -1,153 +0,0 @@ -# Copyright: 2005-2006 Brian Harring <ferringb@gmail.com> -# Copyright: 2006 Marien Zwart <marienz@gentoo.org> -# License: GPL2 - -"""Functions that turn a string into a restriction or raise ParseError. - -@var parse_funcs: dict of the functions that are available. -""" - -from pkgcore.restrictions import packages, values, util -from pkgcore.package import errors -from pkgcore.ebuild import atom, cpv, errors -from snakeoil.containers import InvertedContains - - -class ParseError(ValueError): - """Raised if parsing a restriction expression failed.""" - - -def comma_separated_containment(attr): - """Helper for parsing comma-separated strings to a ContainmentMatch. - - @param attr: name of the attribute. - @returns: a parse function: takes a string of comma-separated values, - returns a L{packages.PackageRestriction} matching packages that - have any of those values in the attribute passed to this function. - """ - def _parse(value): - return packages.PackageRestriction(attr, values.ContainmentMatch(*( - piece.strip() for piece in value.split(',')))) - return _parse - - -def convert_glob(token): - if '*' in token[1:-1]: - raise ParseError( - "'*' must be specified at the end or beginning of a matching field") - l = len(token) - if token.startswith("*") and l > 1: - if token.endswith("*"): - if l == 2: - return None - return values.ContainmentMatch(token.strip("*")) - return values.StrGlobMatch(token.strip("*"), prefix=False) - elif token.endswith("*") and l > 1: - return values.StrGlobMatch(token.strip("*"), prefix=True) - elif l <= 1: - return None - return values.StrExactMatch(token) - -def collect_ops(text): - i = 0 - while text[i] in ("<", "=", ">", "~"): - i += 1 - return text[0:i], text[i:] - -def parse_match(text): - - """generate appropriate restriction for text - - Parsing basically breaks it down into chunks split by /, with each - chunk allowing for prefix/postfix globbing- note that a postfixed - glob on package token is treated as package attribute matching, - B{not} as necessarily a version match. - - If only one chunk is found, it's treated as a package chunk. - Finally, it supports a nonstandard variation of atom syntax where - the category can be dropped. - - Examples- - - "*": match all - - "dev-*/*": category must start with dev- - - "dev-*": package must start with dev- - - *-apps/portage*: category must end in -apps, - package must start with portage - - >=portage-2.1: atom syntax, package portage, - version greater then or equal to 2.1 - - @param text: string to attempt to parse - @type text: string - @return: L{package restriction<pkgcore.restrictions.packages>} derivative - """ - - orig_text = text = text.strip() - if "!" in text: - raise ParseError( - "!, or any form of blockers make no sense in this usage: %s" % ( - text,)) - tsplit = text.rsplit("/", 1) - if len(tsplit) == 1: - ops, text = collect_ops(text) - if not ops: - if "*" in text: - r = convert_glob(text) - if r is None: - return packages.AlwaysTrue - return packages.PackageRestriction("package", r) - elif text.startswith("*"): - raise ParseError( - "cannot do prefix glob matches with version ops: %s" % ( - orig_text,)) - # ok... fake category. whee. - try: - r = list(util.collect_package_restrictions( - atom.atom("%scategory/%s" % (ops, text)).restrictions, - attrs=InvertedContains(["category"]))) - except errors.MalformedAtom, e: - raise ParseError(str(e)) - if len(r) == 1: - return r[0] - return packages.AndRestriction(*r) - elif text[0] in "=<>~": - return atom.atom(text) - if "*" not in text: - try: - return atom.atom(text) - except errors.InvalidCPV, e: - raise ParseError(str(e)) - r = map(convert_glob, tsplit) - if not r[0] and not r[1]: - return packages.AlwaysTrue - if not r[0]: - return packages.PackageRestriction("package", r[1]) - elif not r[1]: - return packages.PackageRestriction("category", r[0]) - return packages.AndRestriction( - packages.PackageRestriction("category", r[0]), - packages.PackageRestriction("package", r[1])) - - -def parse_pv(repo, text): - """Return a CPV instance from either a cpv or a pv string. - - If a pv is passed it needs to match a single cpv in repo. - """ - try: - return cpv.CPV(text) - except errors.InvalidCPV: - restrict = parse_match('=%s' % (text,)) - result = None - for match in repo.itermatch(restrict): - if result is not None: - raise ParseError('multiple matches for %s (%s, %s)' % ( - text, result.cpvstr, match.cpvstr)) - result = match - if result is None: - raise ParseError('no matches for %s' % (text,)) - return cpv.CPV(result.category, result.package, result.version) - - -parse_funcs = { - 'match': parse_match, - } diff --git a/pkgcore/util/repo_utils.py b/pkgcore/util/repo_utils.py deleted file mode 100644 index 2858ac4..0000000 --- a/pkgcore/util/repo_utils.py +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright: 2006 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -""" -misc. repository related tools -""" - -from pkgcore.repository import virtual - -def get_raw_repos(repo): - """ - returns a list of raw repos found. - repo can be either a repo instance, or a list - """ - if isinstance(repo, (list, tuple)): - l = [] - map(l.extend, (get_raw_repos(x) for x in repo)) - return l - while getattr(repo, "raw_repo", None) is not None: - repo = repo.raw_repo - if hasattr(repo, "trees"): - l = [] - map(l.extend, (get_raw_repos(x) for x in repo.trees)) - return l - return [repo] - -def get_virtual_repos(repo, sentinel=True): - """ - select only virtual repos - repo can be either a list, or a repo to descend through. - if sentinel is False, will select all non virtual repos - """ - if not isinstance(repo, (tuple, list)): - repo = get_raw_repos(repo) - return [x for x in repo if isinstance(x, virtual.tree) == sentinel] diff --git a/pkgcore/vdb/__init__.py b/pkgcore/vdb/__init__.py deleted file mode 100644 index 6ffee72..0000000 --- a/pkgcore/vdb/__init__.py +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright: 2005 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -from pkgcore.restrictions.packages import OrRestriction -from pkgcore.repository import multiplex, virtual -from pkgcore.vdb.ondisk import tree as vdb_repository -from snakeoil.currying import partial - -def _grab_virtuals(parent_repo): - virtuals = {} - for pkg in parent_repo: - for virtualpkg in pkg.provides.evaluate_depset(pkg.use): - virtuals.setdefault(virtualpkg.package, {}).setdefault( - pkg.fullver, []).append(pkg) - - for pkg_dict in virtuals.itervalues(): - for full_ver, rdep_atoms in pkg_dict.iteritems(): - if len(rdep_atoms) == 1: - pkg_dict[full_ver] = rdep_atoms[0].unversioned_atom - else: - pkg_dict[full_ver] = OrRestriction( - finalize=True, *[x.unversioned_atom for x in rdep_atoms]) - return virtuals - -def repository(*args, **kwargs): - r = vdb_repository(*args, **kwargs) - return multiplex.tree( - r, virtual.tree(partial(_grab_virtuals, r), livefs=True)) - -repository = vdb_repository diff --git a/pkgcore/vdb/contents.py b/pkgcore/vdb/contents.py deleted file mode 100644 index d82887a..0000000 --- a/pkgcore/vdb/contents.py +++ /dev/null @@ -1,180 +0,0 @@ -# Copyright: 2005-2007 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -from pkgcore.fs.contents import contentsSet -from pkgcore.fs import fs -from pkgcore.interfaces import data_source - -from snakeoil.fileutils import AtomicWriteFile -from snakeoil.compatibility import any -from snakeoil.demandload import demandload -demandload(globals(), - 'os', - 'stat', - 'errno', - 'pkgcore.chksum:get_handler', - 'snakeoil.osutils:readlines', - 'pkgcore:os_data', -) - -class LookupFsDev(fs.fsDev): - - def __init__(self, path, **kwds): - if any(x not in kwds for x in ("major", "minor", "mode")): - try: - st = os.lstat(path) - except OSError, oe: - if oe.errno != errno.ENOENT: - raise - st = None - if st is None or any(f(st.st_mode) for f in - (stat.S_ISREG, stat.S_ISDIR, stat.S_ISFIFO)): - kwds["strict"] = True - else: - major, minor = fs.get_major_minor(st) - kwds["major"] = major - kwds["minor"] = minor - kwds["mode"] = st.st_mode - fs.fsDev.__init__(self, path, **kwds) - - -class ContentsFile(contentsSet): - """class wrapping a contents file""" - - def __init__(self, source, mutable=False, create=False): - - if not isinstance(source, (data_source.base, basestring)): - raise TypeError("source must be either data_source, or a filepath") - contentsSet.__init__(self, mutable=True) - self._source = source - - if not create: - self._read() - - self.mutable = mutable - - def clone(self, empty=False): - # create is used to block it from reading. - cset = self.__class__(self._source, mutable=True, create=True) - if not empty: - cset.update(self) - return cset - - def add(self, obj): - if isinstance(obj, fs.fsFile): - # strict checks - if obj.chksums is None or "md5" not in obj.chksums: - raise TypeError("fsFile objects need to be strict") - elif not isinstance(obj, (fs.fsDir, fs.fsSymlink, fs.fsFifo, fs.fsDev)): - raise TypeError( - "obj must be of fsObj, fsDir, fsLink, fsFifo, fsDev class " - "or derivative") - - contentsSet.add(self, obj) - - def _get_fd(self, write=False): - if isinstance(self._source, basestring): - if write: - return AtomicWriteFile(self._source, uid=os_data.root_uid, - gid=os_data.root_gid, perms=0644) - return readlines(self._source, True) - fobj = self._source.get_fileobj() - if write: - fobj.seek(0, 0) - fobj.truncate(0) - return fobj - - def flush(self): - return self._write() - - def _parse_old(self, line): - """parse old contents, non tab based format""" - # specifically force splitting on spaces. - s = line.split() - if not s: - # stupid; just whitespace/newline. ignore it. - return None - if s[0] in ("dir", "dev", "fif"): - return s[0], ' '.join(s[1:]) - elif s[0] == "obj": - return "obj", ' '.join(s[1:-2]), s[-2], s[-1] - elif s[0] == "sym": - try: - p = s.index("->") - return "sym", ' '.join(s[1:p]), ' '.join(s[p+1:-1]), long(s[-1]) - - except ValueError: - # XXX throw a corruption error - raise - else: - return s[0], ' '.join(s[1:]) - - def _read(self): - self.clear() - for line in self._get_fd(): - line = self._parse_old(line) - if line is None: - continue -# if "\t" not in line: -# line = self._parse_old(line) -# else: -# line = line.split("\t") - - if line[0] == "dir": - obj = fs.fsDir(line[1], strict=False) - elif line[0] == "fif": - obj = fs.fsDir(line[1], strict=False) - elif line[0] == "dev": - obj = LookupFsDev(line[1], strict=False) - elif line[0] == "obj": - #file: path, md5, time - obj = fs.fsFile( - line[1], chksums={"md5":long(line[2], 16)}, - mtime=long(line[3]), - strict=False) - elif line[0] == "sym": - #path, target, ' -> ', mtime - obj = fs.fsLink( - line[1], line[2], mtime=line[3], strict=False) - else: - if len(line) > 2: - line = line[0], ' '.join(line[1:]) - raise Exception( - "unknown entry type %s: %s" % (line[0], line[1])) - self.add(obj) - - def _write(self): - md5_handler = get_handler('md5') - outfile = None - try: - outfile = self._get_fd(True) - - for obj in sorted(self): - - if isinstance(obj, fs.fsFile): - s = " ".join(("obj", obj.location, - md5_handler.long2str(obj.chksums["md5"]), - str(long(obj.mtime)))) - - elif isinstance(obj, fs.fsLink): - s = " ".join(("sym", obj.location, "->", - obj.target, str(long(obj.mtime)))) - - elif isinstance(obj, fs.fsDir): - s = "dir " + obj.location - - elif isinstance(obj, fs.fsDev): - s = "dev " + obj.location - - elif isinstance(obj, fs.fsFifo): - s = "fif " + obj.location - - else: - raise Exception( - "unknown type %s: %s" % (type(obj), obj)) - outfile.write(s + "\n") - outfile.close() - - finally: - # if atomic, it forces the update to be wiped. - del outfile diff --git a/pkgcore/vdb/ondisk.py b/pkgcore/vdb/ondisk.py deleted file mode 100644 index d0964b7..0000000 --- a/pkgcore/vdb/ondisk.py +++ /dev/null @@ -1,201 +0,0 @@ -# Copyright: 2005-2007 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -import os, stat, errno - -from pkgcore.repository import prototype, errors -from pkgcore.vdb import virtuals -from pkgcore.plugin import get_plugin -from pkgcore.interfaces import data_source -from pkgcore.repository import multiplex -from pkgcore.config import ConfigHint -#needed to grab the PN -from pkgcore.ebuild.cpv import CPV as cpv - -from snakeoil.osutils import pjoin -from snakeoil.mappings import IndeterminantDict -from snakeoil.currying import partial -from snakeoil.osutils import listdir_dirs, readfile -from pkgcore.util import bzip2 -from snakeoil.demandload import demandload -demandload(globals(), - 'pkgcore.vdb:repo_ops', - 'pkgcore.vdb.contents:ContentsFile', -) - - -class bz2_data_source(data_source.base): - - def __init__(self, location, mutable=False): - data_source.base.__init__(self) - self.location = location - self.mutable = mutable - - def get_fileobj(self): - data = bzip2.decompress(readfile(self.location)) - if self.mutable: - return data_source.write_StringIO(self._set_data, data) - return data_source.read_StringIO(data) - - def _set_data(self, data): - open(self.location, "wb").write(bzip2.compress(data)) - - -class tree(prototype.tree): - livefs = True - configured = False - configurables = ("domain", "settings") - configure = None - format_magic = "ebuild_built" - - pkgcore_config_type = ConfigHint({'location': 'str', - 'cache_location': 'str', 'repo_id':'str', - 'disable_cache': 'bool'}, typename='repo') - - def __init__(self, location, cache_location=None, repo_id='vdb', - disable_cache=False): - prototype.tree.__init__(self, frozen=False) - self.repo_id = repo_id - self.base = self.location = location - if disable_cache: - cache_location = None - elif cache_location is None: - cache_location = pjoin("/var/cache/edb/dep", - location.lstrip("/")) - self.cache_location = cache_location - self._versions_tmp_cache = {} - try: - st = os.stat(self.base) - if not stat.S_ISDIR(st.st_mode): - raise errors.InitializationError( - "base not a dir: %r" % self.base) - elif not st.st_mode & (os.X_OK|os.R_OK): - raise errors.InitializationError( - "base lacks read/executable: %r" % self.base) - - except OSError: - raise errors.InitializationError( - "lstat failed on base %r" % self.base) - - self.package_class = get_plugin('format.' + self.format_magic)(self) - - def _get_categories(self, *optional_category): - # return if optional_category is passed... cause it's not yet supported - if optional_category: - return {} - try: - try: - return tuple(x for x in listdir_dirs(self.base) if not - x.startswith('.')) - except (OSError, IOError), e: - raise KeyError("failed fetching categories: %s" % str(e)) - finally: - pass - - def _get_packages(self, category): - cpath = pjoin(self.base, category.lstrip(os.path.sep)) - l = set() - d = {} - try: - for x in listdir_dirs(cpath): - if x.startswith(".tmp.") or x.endswith(".lockfile") \ - or x.startswith("-MERGING-"): - continue - x = cpv(category+"/"+x) - l.add(x.package) - d.setdefault((category, x.package), []).append(x.fullver) - except (OSError, IOError), e: - raise KeyError("failed fetching packages for category %s: %s" % \ - (pjoin(self.base, category.lstrip(os.path.sep)), str(e))) - - self._versions_tmp_cache.update(d) - return tuple(l) - - def _get_versions(self, catpkg): - return tuple(self._versions_tmp_cache.pop(catpkg)) - - def _get_ebuild_path(self, pkg): - s = "%s-%s" % (pkg.package, pkg.fullver) - return pjoin(self.base, pkg.category, s, s+".ebuild") - - _metadata_rewrites = { - "depends":"DEPEND", "rdepends":"RDEPEND", "post_rdepends":"PDEPEND", - "use":"USE", "eapi":"EAPI", "CONTENTS":"contents", "provides":"PROVIDE"} - - def _get_metadata(self, pkg): - return IndeterminantDict(partial(self._internal_load_key, - pjoin(self.base, pkg.category, - "%s-%s" % (pkg.package, pkg.fullver)))) - - def _internal_load_key(self, path, key): - key = self._metadata_rewrites.get(key, key) - if key == "contents": - data = ContentsFile(pjoin(path, "CONTENTS"), mutable=True) - elif key == "environment": - fp = pjoin(path, key) - if not os.path.exists(fp+".bz2"): - if not os.path.exists(fp): - # icky. - raise KeyError("environment: no environment file found") - data = data_source.local_source(fp) - else: - data = bz2_data_source(fp+".bz2") - elif key == "ebuild": - fp = pjoin(path, - os.path.basename(path.rstrip(os.path.sep))+".ebuild") - data = data_source.local_source(fp) - else: - data = readfile(pjoin(path, key), True) - if data is None: - raise KeyError(key) - return data - - def notify_remove_package(self, pkg): - remove_it = len(self.packages[pkg.category]) == 1 - prototype.tree.notify_remove_package(self, pkg) - if remove_it: - try: - os.rmdir(pjoin(self.base, pkg.category)) - except OSError, oe: - if oe.errno != errno.ENOTEMPTY: - raise - # silently swallow it; - del oe - - def __str__(self): - return '%s.%s: location %s' % ( - self.__class__.__module__, self.__class__.__name__, self.base) - - -class ConfiguredTree(multiplex.tree): - - livefs = True - - def __init__(self, raw_vdb, domain, domain_settings): - self.domain = domain - self.domain_settings = domain_settings - self.raw_vdb = raw_vdb - if raw_vdb.cache_location is not None: - self.old_style_virtuals = virtuals.caching_virtuals(raw_vdb, - raw_vdb.cache_location) - else: - self.old_style_virtuals = virtuals.non_caching_virtuals(raw_vdb) - multiplex.tree.__init__(self, raw_vdb, self.old_style_virtuals) - self.frozen = raw_vdb.frozen - - def _install(self, pkg, *a, **kw): - # need to verify it's not in already... - kw['offset'] = self.domain.root - return repo_ops.install(self.domain_settings, self.raw_vdb, pkg, *a, **kw) - - def _uninstall(self, pkg, *a, **kw): - kw['offset'] = self.domain.root - return repo_ops.uninstall(self.domain_settings, self.raw_vdb, pkg, *a, **kw) - - def _replace(self, oldpkg, newpkg, *a, **kw): - kw['offset'] = self.domain.root - return repo_ops.replace( - self.domain_settings, self.raw_vdb, oldpkg, newpkg, *a, **kw) - - -tree.configure = ConfiguredTree diff --git a/pkgcore/vdb/repo_ops.py b/pkgcore/vdb/repo_ops.py deleted file mode 100644 index e61be54..0000000 --- a/pkgcore/vdb/repo_ops.py +++ /dev/null @@ -1,157 +0,0 @@ -# Copyright: 2005-2007 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -import os, shutil - -from pkgcore.interfaces import repo as repo_interfaces -#needed to grab the PN - -from pkgcore.const import VERSION - -from snakeoil.osutils import ensure_dirs, pjoin -from pkgcore.util import bzip2 -from snakeoil.demandload import demandload -demandload(globals(), - 'time', - 'pkgcore.ebuild:conditionals', - 'pkgcore.ebuild:triggers', - 'pkgcore.log:logger', - 'pkgcore.fs.ops:change_offset_rewriter', - 'pkgcore.vdb.contents:ContentsFile', -) - - -def _get_default_ebuild_op_args_kwds(self): - return (dict(self.domain_settings),), {} - -def _default_customize_engine(op_inst, engine): - triggers.customize_engine(op_inst.domain_settings, engine) - -class install(repo_interfaces.livefs_install): - - def __init__(self, domain_settings, repo, pkg, *a, **kw): - self.dirpath = pjoin( - repo.base, pkg.category, pkg.package+"-"+pkg.fullver) - self.domain_settings = domain_settings - repo_interfaces.livefs_install.__init__(self, repo, pkg, *a, **kw) - - install_get_format_op_args_kwds = _get_default_ebuild_op_args_kwds - customize_engine = _default_customize_engine - - def merge_metadata(self, dirpath=None): - # error checking? - if dirpath is None: - dirpath = self.dirpath - ensure_dirs(dirpath, mode=0755, minimal=True) - rewrite = self.repo._metadata_rewrites - for k in self.new_pkg.tracked_attributes: - if k == "contents": - v = ContentsFile(pjoin(dirpath, "CONTENTS"), - mutable=True, create=True) - # strip the offset. - if self.offset: - v.update(change_offset_rewriter(self.offset, '/', - self.me.csets["install"])) - else: - v.update(self.me.csets["install"]) - v.flush() - elif k == "environment": - data = bzip2.compress( - self.new_pkg.environment.get_fileobj().read()) - open(pjoin(dirpath, "environment.bz2"), "w").write(data) - del data - else: - v = getattr(self.new_pkg, k) - if k == 'provides': - versionless_providers = lambda b:b.key - s = conditionals.stringify_boolean(v, - func=versionless_providers) - elif not isinstance(v, basestring): - try: - s = ' '.join(v) - except TypeError: - s = str(v) - else: - s = v - open(pjoin( - dirpath, - rewrite.get(k, k.upper())), "w", 32768).write(s) - - # ebuild_data is the actual ebuild- no point in holding onto - # it for built ebuilds, but if it's there, we store it. - o = getattr(self.new_pkg, "ebuild", None) - if o is None: - logger.warn( - "doing install/replace op, " - "but source package doesn't provide the actual ebuild data. " - "Creating an empty file") - o = '' - else: - o = o.get_fileobj().read() - # XXX lil hackish accessing PF - open(pjoin(dirpath, self.new_pkg.PF + ".ebuild"), "w").write(o) - - # XXX finally, hack to keep portage from doing stupid shit. - # relies on counter to discern what to punt during - # merging/removal, we don't need that crutch however. problem? - # No counter file, portage wipes all of our merges (friendly - # bugger). - # need to get zmedico to localize the counter - # creation/counting to per CP for this trick to behave - # perfectly. - open(pjoin(dirpath, "COUNTER"), "w").write(str(int(time.time()))) - - #finally, we mark who made this. - open(pjoin(dirpath, "PKGMANAGER"), "w").write( - "pkgcore-%s" % VERSION) - return True - - -class uninstall(repo_interfaces.livefs_uninstall): - - def __init__(self, domain_settings, repo, pkg, offset=None, *a, **kw): - self.dirpath = pjoin( - repo.base, pkg.category, pkg.package+"-"+pkg.fullver) - self.domain_settings = domain_settings - repo_interfaces.livefs_uninstall.__init__( - self, repo, pkg, offset=offset, *a, **kw) - - uninstall_get_format_op_args_kwds = _get_default_ebuild_op_args_kwds - customize_engine = _default_customize_engine - - def unmerge_metadata(self, dirpath=None): - if dirpath is None: - dirpath = self.dirpath - shutil.rmtree(self.dirpath) - return True - - -# should convert these to mixins. -class replace(install, uninstall, repo_interfaces.livefs_replace): - - def __init__(self, domain_settings, repo, pkg, newpkg, *a, **kw): - self.dirpath = pjoin( - repo.base, pkg.category, pkg.package+"-"+pkg.fullver) - self.newpath = pjoin( - repo.base, newpkg.category, newpkg.package+"-"+newpkg.fullver) - self.tmpdirpath = pjoin( - os.path.dirname(self.dirpath), - ".tmp."+os.path.basename(self.dirpath)) - self.domain_settings = domain_settings - repo_interfaces.livefs_replace.__init__(self, repo, pkg, newpkg, *a, **kw) - - _get_format_op_args_kwds = _get_default_ebuild_op_args_kwds - customize_engine = _default_customize_engine - - def merge_metadata(self, *a, **kw): - kw["dirpath"] = self.tmpdirpath - if os.path.exists(self.tmpdirpath): - shutil.rmtree(self.tmpdirpath) - return install.merge_metadata(self, *a, **kw) - - def unmerge_metadata(self, *a, **kw): - ret = uninstall.unmerge_metadata(self, *a, **kw) - if not ret: - return ret - os.rename(self.tmpdirpath, self.newpath) - return True diff --git a/pkgcore/vdb/virtuals.py b/pkgcore/vdb/virtuals.py deleted file mode 100644 index 040f614..0000000 --- a/pkgcore/vdb/virtuals.py +++ /dev/null @@ -1,182 +0,0 @@ -# Copyright: 2006 Brian Harring <ferringb@gmail.com> -# License: GPL2 - -import os, stat - -from pkgcore.restrictions import packages, values -from pkgcore.ebuild.atom import atom -from pkgcore.package.errors import InvalidDependency -from pkgcore.os_data import portage_gid -from pkgcore.repository import virtual - -from snakeoil.lists import iflatten_instance, unstable_unique -from snakeoil.osutils import listdir, ensure_dirs, pjoin, readlines -from snakeoil.currying import partial -from snakeoil.fileutils import read_dict, AtomicWriteFile -from snakeoil.demandload import demandload -demandload(globals(), "errno") - -# generic functions. - -def _collect_virtuals(virtuals, iterable): - for pkg in iterable: - for virtualpkg in iflatten_instance( - pkg.provides.evaluate_depset(pkg.use)): - virtuals.setdefault(virtualpkg.package, {}).setdefault( - pkg.fullver, []).append(pkg.versioned_atom) - -def _finalize_virtuals(virtuals): - for pkg_dict in virtuals.itervalues(): - for full_ver, rdep_atoms in pkg_dict.iteritems(): - pkg_dict[full_ver] = tuple(rdep_atoms) - -def _collect_default_providers(virtuals): - return dict((virt, - frozenset(atom(x.key) for y in data.itervalues() for x in y)) - for virt, data in virtuals.iteritems()) - -# noncaching... - -def _grab_virtuals(repo): - virtuals = {} - _collect_virtuals(virtuals, repo) - defaults = _collect_default_providers(virtuals) - _finalize_virtuals(virtuals) - return defaults, virtuals - -def non_caching_virtuals(repo, livefs=True): - return OldStyleVirtuals(partial(_grab_virtuals, repo)) - - -#caching - -def _get_mtimes(loc): - d = {} - sdir = stat.S_ISDIR - for x in listdir(loc): - st = os.stat(pjoin(loc, x)) - if sdir(st.st_mode): - d[x] = st.st_mtime - return d - -def _write_mtime_cache(mtimes, data, location): - old = os.umask(0113) - try: - if not ensure_dirs(os.path.dirname(location), - gid=portage_gid, mode=0775): - # bugger, can't update.. - return - f = AtomicWriteFile(location, gid=portage_gid, perms=0664) - # invert the data... - rev_data = {} - for pkg, ver_dict in data.iteritems(): - for fullver, virtuals in ver_dict.iteritems(): - for virtual in virtuals: - rev_data.setdefault(virtual.category, []).extend( - (pkg, fullver, str(virtual))) - for cat, mtime in mtimes.iteritems(): - if cat in rev_data: - f.write("%s\t%i\t%s\n" % (cat, mtime, - '\t'.join(rev_data[cat]))) - else: - f.write("%s\t%i\n" % (cat, mtime)) - f.close() - del f - finally: - os.umask(old) - os.chown(location, -1, portage_gid) - -def _read_mtime_cache(location): - try: - d = {} - for k, v in read_dict(readlines(location), splitter=None, - source_isiter=True).iteritems(): - v = v.split() - # mtime pkg1 fullver1 virtual1 pkg2 fullver2 virtual2... - # if it's not the right length, skip this entry, - # cache validation will update it. - if (len(v) -1) % 3 == 0: - d[k] = v - return d - except IOError, e: - if e.errno != errno.ENOENT: - raise - return {} - -def _convert_cached_virtuals(data): - iterable = iter(data) - # skip the mtime entry. - iterable.next() - d = {} - try: - for item in iterable: - d.setdefault(item, {}).setdefault(iterable.next(), []).append( - atom(iterable.next())) - except InvalidDependency: - return None - return d - -def _merge_virtuals(virtuals, new_virts): - for pkg, fullver_d in new_virts.iteritems(): - for fullver, provides in fullver_d.iteritems(): - virtuals.setdefault(pkg, {}).setdefault( - fullver, []).extend(provides) - -def _caching_grab_virtuals(repo, cache_basedir): - virtuals = {} - update = False - cache = _read_mtime_cache(pjoin(cache_basedir, 'virtuals.cache')) - - existing = _get_mtimes(repo.location) - for cat, mtime in existing.iteritems(): - d = cache.pop(cat, None) - if d is not None and long(d[0]) == mtime: - d = _convert_cached_virtuals(d) - if d is not None: - _merge_virtuals(virtuals, d) - continue - - update = True - _collect_virtuals(virtuals, repo.itermatch( - packages.PackageRestriction("category", - values.StrExactMatch(cat)))) - - if update or cache: - _write_mtime_cache(existing, virtuals, - pjoin(cache_basedir, 'virtuals.cache')) - - defaults = _collect_default_providers(virtuals) -# _finalize_virtuals(virtuals) - return defaults, virtuals - -def caching_virtuals(repo, cache_basedir, livefs=True): - return OldStyleVirtuals(partial(_caching_grab_virtuals, repo, cache_basedir)) - - -class OldStyleVirtuals(virtual.tree): - - def __init__(self, load_func): - virtual.tree.__init__(self, livefs=True) - self._load_func = load_func - - def _load_data(self): - self.default_providers, self._virtuals = self._load_func() - self.packages._cache['virtual'] = tuple(self._virtuals.iterkeys()) - self.versions._cache.update((('virtual', k), tuple(ver_dict)) - for k, ver_dict in self._virtuals.iteritems()) - self.versions._finalized = True - self.versions._known_keys.clear() - self._load_func = None - - def _expand_vers(self, cp, ver): - return self._virtuals[cp[1]][ver] - - def __getattr__(self, attr): - if attr not in ('default_providers', '_virtuals'): - return virtual.tree.__getattr__(self, attr) - if self._load_func is not None: - self._load_data() - return getattr(self, attr) - - def _get_versions(self, cp): - return tuple(self._virtuals[cp[1]].iterkeys()) diff --git a/pkgcore/version.py b/pkgcore/version.py deleted file mode 100644 index 0be7523..0000000 --- a/pkgcore/version.py +++ /dev/null @@ -1,57 +0,0 @@ -# Copyright: 2006 Marien Zwart <marienz@gentoo.org> -# License: GPL2 - - -"""Version information (tied to bzr).""" - - -from pkgcore import const - - -_ver = None - - -def get_version(): - """@returns: a string describing the pkgcore version.""" - global _ver - if _ver is not None: - return _ver - - # This should get overwritten below, but let's be paranoid. - rev = 'unknown revision (internal error)' - version_info = None - try: - from pkgcore.bzr_verinfo import version_info - except ImportError: - try: - from bzrlib import branch, errors - except ImportError: - rev = 'unknown revision ' \ - '(not from an sdist tarball, bzr unavailable)' - else: - try: - # Returns a (branch, relpath) tuple, ignore relpath. - b = branch.Branch.open_containing(__file__)[0] - except errors.NotBranchError: - rev = 'unknown revision ' \ - '(not from an sdist tarball, not a bzr branch)' - else: - version_info = { - 'branch_nick': b.nick, - 'revno': b.revno(), - 'revision_id': b.last_revision(), - } - if b.supports_tags(): - tagdict = b.tags.get_reverse_tag_dict() - version_info['tags'] = tagdict.get(b.last_revision()) - if version_info is not None: - tags = version_info.get('tags') - if tags: - revname = ' '.join('tag:%s' % (tag,) for tag in tags) - else: - revname = '%(revno)s revid:%(revision_id)s' % version_info - rev = 'from bzr branch %s %s' % (version_info['branch_nick'], revname) - - _ver = 'pkgcore %s\n%s' % (const.VERSION, rev) - - return _ver |