cas | 19 ++-
cas-admin | 2
lib/cas/cas_shutil.py | 268 ++++++++++++++++++++++++++++++++++++++++++++++++++
lib/cas/util.py | 2
version | 2
5 files changed, 285 insertions(+), 8 deletions(-)
New commits:
commit 4fe82903020250a62deb08e017fa2e6b080fb5ff
Author: Adam Stokes <ajs(a)redhat.com>
Date: Tue Dec 16 10:10:03 2008 -0500
- include shutil from python 2.6, major performance
improvement within the move method
- decided to handle all files within the `cwd` until
extracted and ready for processing (thats where shutil
comes into play)
diff --git a/cas b/cas
index 0e66cda..2854f72 100755
--- a/cas
+++ b/cas
@@ -18,7 +18,6 @@ import sys
import optparse
import os
import ConfigParser
-import shutil
import smtplib
from subprocess import Popen, PIPE
@@ -28,6 +27,10 @@ from cas.core import CoreBase, CoreException
from cas.util import UtilBase, Logging
from cas.rpmutils import RPMBase
+# shutil from python 2.6 includes several enhancements for our use
+# mainly in the _move_ method.
+import cas.cas_shutil as shutil
+
if sys.version_info[:2] < (2,4):
raise SystemExit("Python >= 2.4 required")
@@ -43,6 +46,7 @@ SMTPHOST = config.get("settings", "mailServer")
class CoreHandler(object):
def __init__(self, filename, dst, logger):
self.filename = filename
+ self.basename = os.path.basename(self.filename)
self.dst = dst
self.currentDirectory = os.path.realpath(os.curdir)
self.casLog = logger
@@ -61,17 +65,22 @@ class CoreHandler(object):
self.casLog.debug("Unable to find file %s" % (self.filename,))
sys.exit(1)
if self.tool.isCorefile(self.filename):
- basename = os.path.basename(self.filename)
# No need to proceed to extracting corefile since we assume
# this is already at the proper stage.
shutil.move(self.filename,
- os.path.join(self.dst, basename))
- self.filename = os.path.join(self.dst, basename)
+ os.path.join(self.dst, self.basename))
+ self.filename = os.path.join(self.dst, self.basename)
return self.filename
try:
self.casLog.info("Detected a compressed core, extracting.. please wait as " \
"this process can take a long time.")
- corepath = self.tool.extractCore(self.filename)
+ # ok so some decompression utilites like gzip, bzip do not extract
+ # files into the `cwd` unless specified with a to-stdout option.
+ # this is a pain so we just move everything to `cwd` and proceed
+ # from there.
+ dst = os.path.join(self.currentDirectory, self.basename)
+ shutil.move(self.filename, dst)
+ corepath = self.tool.extractCore(dst)
# corefile extracted now move it to work directory, pull basename
# from corepath since we auto-detect the core file from extraction
coreBasename = os.path.basename(corepath)
diff --git a/cas-admin b/cas-admin
index a852be0..9e1c553 100755
--- a/cas-admin
+++ b/cas-admin
@@ -24,7 +24,7 @@ from cas.core import CoreBase
from cas.util import UtilBase, Logging
from cas.rpmutils import RPMBase
from subprocess import Popen, PIPE
-from shutil import rmtree
+from cas.cas_shutil import rmtree
if sys.version_info[:2] < (2,4):
raise SystemExit("Python >= 2.4 required")
diff --git a/lib/cas/cas_shutil.py b/lib/cas/cas_shutil.py
new file mode 100644
index 0000000..ae2c66c
--- /dev/null
+++ b/lib/cas/cas_shutil.py
@@ -0,0 +1,268 @@
+"""Utility functions for copying files and directory trees.
+
+XXX The functions here don't copy the resource fork or other metadata on Mac.
+
+"""
+
+import os
+import sys
+import stat
+from os.path import abspath
+import fnmatch
+
+__all__ = ["copyfileobj","copyfile","copymode","copystat","copy","copy2",
+ "copytree","move","rmtree","Error"]
+
+class Error(EnvironmentError):
+ pass
+
+try:
+ WindowsError
+except NameError:
+ WindowsError = None
+
+def copyfileobj(fsrc, fdst, length=16*1024):
+ """copy data from file-like object fsrc to file-like object fdst"""
+ while 1:
+ buf = fsrc.read(length)
+ if not buf:
+ break
+ fdst.write(buf)
+
+def _samefile(src, dst):
+ # Macintosh, Unix.
+ if hasattr(os.path,'samefile'):
+ try:
+ return os.path.samefile(src, dst)
+ except OSError:
+ return False
+
+ # All other platforms: check for same pathname.
+ return (os.path.normcase(os.path.abspath(src)) ==
+ os.path.normcase(os.path.abspath(dst)))
+
+def copyfile(src, dst):
+ """Copy data from src to dst"""
+ if _samefile(src, dst):
+ raise Error, "`%s` and `%s` are the same file" % (src, dst)
+
+ fsrc = None
+ fdst = None
+ try:
+ fsrc = open(src, 'rb')
+ fdst = open(dst, 'wb')
+ copyfileobj(fsrc, fdst)
+ finally:
+ if fdst:
+ fdst.close()
+ if fsrc:
+ fsrc.close()
+
+def copymode(src, dst):
+ """Copy mode bits from src to dst"""
+ if hasattr(os, 'chmod'):
+ st = os.stat(src)
+ mode = stat.S_IMODE(st.st_mode)
+ os.chmod(dst, mode)
+
+def copystat(src, dst):
+ """Copy all stat info (mode bits, atime, mtime, flags) from src to dst"""
+ st = os.stat(src)
+ mode = stat.S_IMODE(st.st_mode)
+ if hasattr(os, 'utime'):
+ os.utime(dst, (st.st_atime, st.st_mtime))
+ if hasattr(os, 'chmod'):
+ os.chmod(dst, mode)
+ if hasattr(os, 'chflags') and hasattr(st, 'st_flags'):
+ os.chflags(dst, st.st_flags)
+
+
+def copy(src, dst):
+ """Copy data and mode bits ("cp src dst").
+
+ The destination may be a directory.
+
+ """
+ if os.path.isdir(dst):
+ dst = os.path.join(dst, os.path.basename(src))
+ copyfile(src, dst)
+ copymode(src, dst)
+
+def copy2(src, dst):
+ """Copy data and all stat info ("cp -p src dst").
+
+ The destination may be a directory.
+
+ """
+ if os.path.isdir(dst):
+ dst = os.path.join(dst, os.path.basename(src))
+ copyfile(src, dst)
+ copystat(src, dst)
+
+def ignore_patterns(*patterns):
+ """Function that can be used as copytree() ignore parameter.
+
+ Patterns is a sequence of glob-style patterns
+ that are used to exclude files"""
+ def _ignore_patterns(path, names):
+ ignored_names = []
+ for pattern in patterns:
+ ignored_names.extend(fnmatch.filter(names, pattern))
+ return set(ignored_names)
+ return _ignore_patterns
+
+def copytree(src, dst, symlinks=False, ignore=None):
+ """Recursively copy a directory tree using copy2().
+
+ The destination directory must not already exist.
+ If exception(s) occur, an Error is raised with a list of reasons.
+
+ If the optional symlinks flag is true, symbolic links in the
+ source tree result in symbolic links in the destination tree; if
+ it is false, the contents of the files pointed to by symbolic
+ links are copied.
+
+ The optional ignore argument is a callable. If given, it
+ is called with the `src` parameter, which is the directory
+ being visited by copytree(), and `names` which is the list of
+ `src` contents, as returned by os.listdir():
+
+ callable(src, names) -> ignored_names
+
+ Since copytree() is called recursively, the callable will be
+ called once for each directory that is copied. It returns a
+ list of names relative to the `src` directory that should
+ not be copied.
+
+ XXX Consider this example code rather than the ultimate tool.
+
+ """
+ names = os.listdir(src)
+ if ignore is not None:
+ ignored_names = ignore(src, names)
+ else:
+ ignored_names = set()
+
+ os.makedirs(dst)
+ errors = []
+ for name in names:
+ if name in ignored_names:
+ continue
+ srcname = os.path.join(src, name)
+ dstname = os.path.join(dst, name)
+ try:
+ if symlinks and os.path.islink(srcname):
+ linkto = os.readlink(srcname)
+ os.symlink(linkto, dstname)
+ elif os.path.isdir(srcname):
+ copytree(srcname, dstname, symlinks, ignore)
+ else:
+ copy2(srcname, dstname)
+ # XXX What about devices, sockets etc.?
+ except (IOError, os.error), why:
+ errors.append((srcname, dstname, str(why)))
+ # catch the Error from the recursive copytree so that we can
+ # continue with other files
+ except Error, err:
+ errors.extend(err.args[0])
+ try:
+ copystat(src, dst)
+ except OSError, why:
+ if WindowsError is not None and isinstance(why, WindowsError):
+ # Copying file access times may fail on Windows
+ pass
+ else:
+ errors.extend((src, dst, str(why)))
+ if errors:
+ raise Error, errors
+
+def rmtree(path, ignore_errors=False, onerror=None):
+ """Recursively delete a directory tree.
+
+ If ignore_errors is set, errors are ignored; otherwise, if onerror
+ is set, it is called to handle the error with arguments (func,
+ path, exc_info) where func is os.listdir, os.remove, or os.rmdir;
+ path is the argument to that function that caused it to fail; and
+ exc_info is a tuple returned by sys.exc_info(). If ignore_errors
+ is false and onerror is None, an exception is raised.
+
+ """
+ if ignore_errors:
+ def onerror(*args):
+ pass
+ elif onerror is None:
+ def onerror(*args):
+ raise
+ try:
+ if os.path.islink(path):
+ # symlinks to directories are forbidden, see bug #1669
+ raise OSError("Cannot call rmtree on a symbolic link")
+ except OSError:
+ onerror(os.path.islink, path, sys.exc_info())
+ # can't continue even if onerror hook returns
+ return
+ names = []
+ try:
+ names = os.listdir(path)
+ except os.error, err:
+ onerror(os.listdir, path, sys.exc_info())
+ for name in names:
+ fullname = os.path.join(path, name)
+ try:
+ mode = os.lstat(fullname).st_mode
+ except os.error:
+ mode = 0
+ if stat.S_ISDIR(mode):
+ rmtree(fullname, ignore_errors, onerror)
+ else:
+ try:
+ os.remove(fullname)
+ except os.error, err:
+ onerror(os.remove, fullname, sys.exc_info())
+ try:
+ os.rmdir(path)
+ except os.error:
+ onerror(os.rmdir, path, sys.exc_info())
+
+
+def _basename(path):
+ # A basename() variant which first strips the trailing slash, if present.
+ # Thus we always get the last component of the path, even for directories.
+ return os.path.basename(path.rstrip(os.path.sep))
+
+def move(src, dst):
+ """Recursively move a file or directory to another location. This is
+ similar to the Unix "mv" command.
+
+ If the destination is a directory or a symlink to a directory, the source
+ is moved inside the directory. The destination path must not already
+ exist.
+
+ If the destination already exists but is not a directory, it may be
+ overwritten depending on os.rename() semantics.
+
+ If the destination is on our current filesystem, then rename() is used.
+ Otherwise, src is copied to the destination and then removed.
+ A lot more could be done here... A look at a mv.c shows a lot of
+ the issues this implementation glosses over.
+
+ """
+ real_dst = dst
+ if os.path.isdir(dst):
+ real_dst = os.path.join(dst, _basename(src))
+ if os.path.exists(real_dst):
+ raise Error, "Destination path '%s' already exists" % real_dst
+ try:
+ os.rename(src, real_dst)
+ except OSError:
+ if os.path.isdir(src):
+ if destinsrc(src, dst):
+ raise Error, "Cannot move a directory '%s' into itself '%s'." % (src, dst)
+ copytree(src, real_dst, symlinks=True)
+ rmtree(src)
+ else:
+ copy2(src, real_dst)
+ os.unlink(src)
+
+def destinsrc(src, dst):
+ return abspath(dst).startswith(abspath(src))
diff --git a/lib/cas/util.py b/lib/cas/util.py
index d17232c..3ed4fff 100755
--- a/lib/cas/util.py
+++ b/lib/cas/util.py
@@ -16,8 +16,8 @@ import commands
import os
import re
import sys
-import shutil
import logging
+import cas.cas_shutil as shutil
from subprocess import Popen, PIPE, call
diff --git a/version b/version
index 094b0d8..7885ba9 100644
--- a/version
+++ b/version
@@ -1 +1 @@
-0.13 107
+0.13 110
