cas September 2009

cas@lists.stg.fedorahosted.org

1 participants
7 discussions

cas cas-admin cas.conf lib/cas README setup.py
by Adam Stokes 28 Sep '09

28 Sep '09

README | 9 +-------- cas | 4 +++- cas-admin | 44 +++++++++++++++++--------------------------- cas.conf | 3 +++ lib/cas/network.py | 6 ++++-- lib/cas/rpmutils.py | 1 - setup.py | 2 +- 7 files changed, 29 insertions(+), 40 deletions(-) New commits: commit 42b1661e595c94fc19d5384d878deef5bbf82e35 Author: adam stokes <uzr(a)jak.rebo> Date: Mon Sep 28 14:58:41 2009 -0400 - more paramiko integration diff --git a/README b/README index df865b7..dc8ed76 100644 --- a/README +++ b/README @@ -14,14 +14,7 @@ As root run: # cas-admin -b -Optional Requirements - -func - Please obtain this software from http://fedorahosted.org/func - setup is -a breeze and enables cas to do automated core processing on different machine -for different architectures. - -If you have func installed and setup you may re-run cas-admin to populate -your server database: +Build your server database: # cas-admin -s diff --git a/cas b/cas index 9423bd7..3e3ade1 100755 --- a/cas +++ b/cas @@ -273,7 +273,9 @@ class CasApplication(object): "crash output" % (casProcessMachine,)) cmd = "cd " + os.path.join(self.storagePath) + " && ./crash -i crash.in > crash.out" # TODO: run command through paramiko - cmd_return = Executor(hostname, "root").run(cmd) + results, err = Executor(hostname, settings["CASUSER"], cmd).run() + if err: + self.casLog.debug("Unable to perform results") else: self.casLog.info("No servers available for arch and current system not "\ "suitable for processing, please run cas-admin -h " \ diff --git a/cas-admin b/cas-admin index 2558f7b..63d434d 100755 --- a/cas-admin +++ b/cas-admin @@ -109,33 +109,20 @@ class CasServerHandler(object): self.db.buildTable() def run(self): - try: - serverList = {} - hostname_count = 0 - # TODO: setup paramiko to run through hostname lists and detect archicture. - for i in minions: - scheme, netloc, path, query, frag = urlparse.urlsplit(i) - hostname, port = netloc.split(":") - client = fc.Overlord(hostname) - isClientReachable = client.run("test","ping", []) - if isClientReachable == 1: - # Returns as {"hostname":['sts','stdout','stderr']} - # what a pain - client_dict = client.command.run("/bin/uname -m") - # silly magic to get output from command module - # since we are doing one host at a time we just pull the first - # key from the dict and parse its return - sts, arch, err = client_dict[client_dict.keys()[0]] - if sts: - self.casLog.debug(err) - # clean up arch string - arch = arch.strip() - self.db.addServer(hostname, arch) - hostname_count = hostname_count + 1 - self.casLog.info("Server database built with %d server(s) added." % (hostname_count,)) - except ImportError: - raise SystemExit(self.casLog.debug("Please install func (http://fedorahosted.org/func) for " \ - "an automated machine arch population.\n")) + hostname_count = 0 + if self.server_list and os.path.isfile(self.server_list): + fd = open(self.server_list, 'r') + for server in fd.readlines(): + server = server.strip() + results, err = Executor(server, settings["CASUSER"], "/bin/uname -m").run() + if err: + self.casLog.debug(err) + # clean up arch string + for i in results: + arch = i.strip() + self.db.addServer(server, arch) + hostname_count = hostname_count + 1 + self.casLog.info("Server database built with %d server(s) added." % (hostname_count,)) return class PurgeHandler(object): @@ -179,6 +166,8 @@ class CasAdminApplication(object): parser.add_option("-s","--server", dest="server_init", help="Build SERVER DB", action="store_true", default=False) + parser.add_option("-f", "--serverlist", dest="server_list", + help="Read in line delimeted server list for cas to propagate") parser.add_option("-p","--purge", dest="purgeData", help="Purge files default 90 days, customize with -d", action="store_true", default=False) @@ -187,6 +176,7 @@ class CasAdminApplication(object): (self.opts, args) = parser.parse_args() self.buildDB = self.opts.buildDB self.server_init = self.opts.server_init + self.server_list = self.opts.server_list self.purgeData = self.opts.purgeData self.purgeDataDays = self.opts.purgeDataDays diff --git a/cas.conf b/cas.conf index b3dcd8d..f3498d6 100644 --- a/cas.conf +++ b/cas.conf @@ -10,6 +10,9 @@ # kernels, rpmFilter, debugs, workDirectory, database [settings] +# Set the user you wish to run cas as. +casuser=root + # Where kernel-debuginfo packages are stored # NOTE: this _can_ include symlinked directories, just be careful they are # indefinately recursive diff --git a/lib/cas/network.py b/lib/cas/network.py index 7be14ab..842863c 100644 --- a/lib/cas/network.py +++ b/lib/cas/network.py @@ -70,7 +70,7 @@ class Executor(object): """ using paramiko to perform common tasks over ssh """ - def __init__(self, user, remote_host, cmd): + def __init__(self, remote_host, user, cmd): self.user = user self.host = remote_host self.cmd = cmd @@ -80,4 +80,6 @@ class Executor(object): def run(self): stdin, stdout, stderr = self.client.exec_command(self.cmd) - return (stdin, stdout, stderr) + results = stdout.readlines() + err = stderr.readlines() + return (results, err) diff --git a/lib/cas/rpmutils.py b/lib/cas/rpmutils.py index 5a65371..da5dfcb 100644 --- a/lib/cas/rpmutils.py +++ b/lib/cas/rpmutils.py @@ -15,7 +15,6 @@ import rpm import os import sys import logging -import md5 import time from cas.cas_subprocess import Popen, PIPE diff --git a/setup.py b/setup.py index d5aba5b..16773ff 100644 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ for dirpath, dirnames, filenames in os.walk('snippets'): setup( name = 'cas', - version = '0.14', + version = '0.15', author = 'Adam Stokes', author_email = 'ajs(a)redhat.com', url = "http://fedorahosted.org/cas",

1 0

cas cas-admin cas.conf
by Adam Stokes 28 Sep '09

28 Sep '09

cas | 17 ++++++++--------- cas-admin | 34 ++++++++++++++++------------------ cas.conf | 2 +- 3 files changed, 25 insertions(+), 28 deletions(-) New commits: commit 62c37e64f4049c0fbf0d88ebc5bdfc785edcbede Author: adam stokes <uzr(a)jak.rebo> Date: Mon Sep 28 13:34:37 2009 -0400 - rework config items to be in a dict allows for easy addition of configuration items diff --git a/cas b/cas index 87519aa..9423bd7 100755 --- a/cas +++ b/cas @@ -42,11 +42,10 @@ if sys.version_info[:2] < (2,3): # into a key,value pair automatically config = ConfigParser.ConfigParser() config.read("/etc/cas.conf") -WORKDIRECTORY = config.get("settings","workDirectory") -DEBUGLEVEL = config.get("settings","debugLevel") -SMTPHOST = config.get("settings", "mailServer") -# Read database parameters -DATABASE = config.get("settings", "database") +settings = {} +if config.has_section("settings"): + for opt, val in config.items("settings"): + settings[opt.upper()] = val # Check for some advanced configurations # Test to see if we provide a 32bit crash binary @@ -134,7 +133,7 @@ class TimestampHandler(object): class CasApplication(object): def __init__(self, args): - self.db = CasStorage(DATABASE) + self.db = CasStorage(settings["DATABASE"]) self.db.connect() self.parse_options(args) self.util = UtilBase() @@ -173,7 +172,7 @@ class CasApplication(object): # build logger object to deal with logging per job and keep things # clean and easy to debug - self.casLog = Logging(self.storagePath, self.identifier, DEBUGLEVEL) + self.casLog = Logging(self.storagePath, self.identifier, settings["DEBUGLEVEL"]) # Add job to database, this allows us to purge # data based on job date creation then by other @@ -321,7 +320,7 @@ class CasApplication(object): if os.path.isfile(crashOutFile) and self.email: self.casLog.info("Crash output processed, sending email to %s" % (self.email,)) try: - mailServer = smtplib.SMTP(SMTPHOST) + mailServer = smtplib.SMTP(settings["SMTPHOST"]) try: # Compose email msg of results msg = "Subject: CAS results for %s\r\n\n" % (self.identifier,) @@ -346,7 +345,7 @@ class CasApplication(object): if __name__=="__main__": # Before we start the database has to exist. - if not os.path.isfile(DATABASE): + if not os.path.isfile(settings["DATABASE"]): raise SystemExit("No Database exist, please run cas-admin --help for more information.") # Begin CAS app = CasApplication(sys.argv[1:]) diff --git a/cas-admin b/cas-admin index 728a6f5..2558f7b 100755 --- a/cas-admin +++ b/cas-admin @@ -36,12 +36,10 @@ if sys.version_info[:2] < (2,3): # Read in configuration config = ConfigParser.ConfigParser() config.read("/etc/cas.conf") -KERNELS = config.get("settings","kernels") -RPMFILTER = config.get("settings","rpmFilter") -DEBUGS = config.get("settings","debugs") -DEBUGLEVEL = config.get("settings","debugLevel") -DATABASE = config.get("settings","database") -WORKDIRECTORY = config.get("settings","workDirectory") +settings = {} +if config.has_section("settings"): + for opt, val in config.items("settings"): + settings[opt.upper()] = val # read maintenance options PURGELIMIT = config.get("maintenance","purgeLimit") @@ -53,9 +51,9 @@ class CasDatabaseHandler(object): self.util = UtilBase() # setup database connection self.first_run = False - if not os.path.isfile(DATABASE): + if not os.path.isfile(settings["DATABASE"]): self.first_run = True - self.db = CasStorage(DATABASE) + self.db = CasStorage(settings["DATABASE"]) self.db.connect() if self.first_run: self.db.buildTable() @@ -64,7 +62,7 @@ class CasDatabaseHandler(object): # Uses emacs regex -- see `man find` # TODO: rework kernel filtering to expand scope # beyond just debuginfo rpms. - cmd = ["find", "-L", KERNELS, "-iregex", RPMFILTER] + cmd = ["find", "-L", settings["KERNELS"], "-iregex", settings["RPMFILTER"]] pipe = Popen(cmd, stdout=PIPE, stderr=PIPE) # setup count for kernels found, mainly for console output count = 0 @@ -77,7 +75,7 @@ class CasDatabaseHandler(object): rpms = self.db.getAllDebuginfoRPM() for id, rpm in rpms: # temporary storage path in form of DEBUGS/COUNT - dst = os.path.join(DEBUGS, str(count)) + dst = os.path.join(settings["DEBUGS"], str(count)) if not os.path.isdir(dst): os.makedirs(dst) rpmTool = RPMBase() @@ -103,9 +101,9 @@ class CasServerHandler(object): self.util = UtilBase() # setup database connection self.first_run = False - if not os.path.isfile(DATABASE): + if not os.path.isfile(settings["DATABASE"]): self.first_run = True - self.db = CasStorage(DATABASE) + self.db = CasStorage(settings["DATABASE"]) self.db.connect() if self.first_run: self.db.buildTable() @@ -152,7 +150,7 @@ class PurgeHandler(object): cutOffDate = self.todaysDate - datetime.timedelta(days=self.purgeDataDays) self.casLog.debug(cutOffDate) # Start of purging data - for root, dirs, files in os.walk(WORKDIRECTORY): + for root, dirs, files in os.walk(settings["WORKDIRECTORY"]): for d in dirs: dirpath = os.path.join(root,d) # pull out date from directory structure and trim it to @@ -171,7 +169,7 @@ class PurgeHandler(object): class CasAdminApplication(object): def __init__(self, args): self.parse_options(args) - self.casLog = Logging("/var/log","cas-admin", DEBUGLEVEL) + self.casLog = Logging("/var/log","cas-admin", settings["DEBUGLEVEL"]) def parse_options(self, args): parser = optparse.OptionParser(usage="cas-admin [opts] args") @@ -198,10 +196,10 @@ class CasAdminApplication(object): """ if os.getuid() is not 0: raise RuntimeError, "You must be root(0), instead you are id(%d)" % (os.getuid()) - if not os.path.isdir(os.path.dirname(DATABASE)): - os.makedirs(os.path.dirname(DATABASE)) - if not os.path.isdir(DEBUGS): - os.makedirs(DEBUGS) + if not os.path.isdir(os.path.dirname(settings["DATABASE"])): + os.makedirs(os.path.dirname(settings["DATABASE"])) + if not os.path.isdir(settings["DEBUGS"]): + os.makedirs(settings["DEBUGS"]) # if autopurge is enabled lets clean up some stale data if AUTOPURGE: diff --git a/cas.conf b/cas.conf index 1928139..b3dcd8d 100644 --- a/cas.conf +++ b/cas.conf @@ -34,7 +34,7 @@ workDirectory=/cores # Mail server, e.g. mail.example.com # Provides job results via email -mailServer=mail.example.com +stmphost=mail.example.com # database connection info database=/var/db/cas/cas.db

1 0

2 commits - cas cas-admin cas.spec lib/cas
by Adam Stokes 28 Sep '09

28 Sep '09

cas | 3 ++- cas-admin | 6 ++---- cas.spec | 12 ++++++++---- lib/cas/network.py | 17 +++++++++++++++++ 4 files changed, 29 insertions(+), 9 deletions(-) New commits: commit e74048977697933ea595a8b97e4b327fd3275b6d Author: adam stokes <uzr(a)jak.rebo> Date: Mon Sep 28 12:26:10 2009 -0400 - start reworking ssh code to use paramiko instead of func diff --git a/cas b/cas index f4e3f9d..87519aa 100755 --- a/cas +++ b/cas @@ -24,7 +24,7 @@ import socket from cas.cas_subprocess import Popen, PIPE from datetime import datetime -from cas.network import Download, CasNetworkException +from cas.network import Download, Executor, CasNetworkException from cas.core import CoreBase, CoreException from cas.db import CasStorage, CasStorageException from cas.util import UtilBase, Logging @@ -274,6 +274,7 @@ class CasApplication(object): "crash output" % (casProcessMachine,)) cmd = "cd " + os.path.join(self.storagePath) + " && ./crash -i crash.in > crash.out" # TODO: run command through paramiko + cmd_return = Executor(hostname, "root").run(cmd) else: self.casLog.info("No servers available for arch and current system not "\ "suitable for processing, please run cas-admin -h " \ diff --git a/cas-admin b/cas-admin index 2e575d8..728a6f5 100755 --- a/cas-admin +++ b/cas-admin @@ -28,6 +28,7 @@ from cas.rpmutils import RPMBase from cas.db import CasStorage, CasStorageException from cas.cas_subprocess import Popen, PIPE from cas.cas_shutil import rmtree +from cas.network import Executor if sys.version_info[:2] < (2,3): raise SystemExit("Python >= 2.3 required") @@ -113,10 +114,7 @@ class CasServerHandler(object): try: serverList = {} hostname_count = 0 - # Attempt to load func to automate detection of machine arch - import func.overlord.client as fc - parent_func = fc.Overlord("*") - minions = parent_func.minions + # TODO: setup paramiko to run through hostname lists and detect archicture. for i in minions: scheme, netloc, path, query, frag = urlparse.urlsplit(i) hostname, port = netloc.split(":") diff --git a/cas.spec b/cas.spec index 708bbcc..cc57342 100644 --- a/cas.spec +++ b/cas.spec @@ -12,9 +12,9 @@ BuildArch: noarch Url: http://fedorahosted.org/cas BuildRequires: python-devel %if 0%{?rhel} -Requires: python-sqlite crash PyYAML +Requires: python-sqlite crash python-paramiko %else -Requires: crash python-paramiko PyYAML +Requires: crash python-paramiko %endif %description @@ -50,7 +50,7 @@ rm -rf ${RPM_BUILD_ROOT} %changelog * Tue Sep 15 2009 Adam Stokes <ajs at redhat dot com> - 0.15-1 -- Require PyYAML, paramiko +- Require paramiko for all remote executions - Rip out func code * Tue May 5 2009 Adam Stokes <ajs at redhat dot com> - 0.14-8 diff --git a/lib/cas/network.py b/lib/cas/network.py index a14d9ff..7be14ab 100644 --- a/lib/cas/network.py +++ b/lib/cas/network.py @@ -15,6 +15,7 @@ import sys import os import urlparse import urlgrabber.grabber as grabber +import paramiko class CasNetworkException(Exception): pass @@ -64,3 +65,19 @@ class Download(object): f.close() url.close() return self.output + +class Executor(object): + """ using paramiko to perform common tasks over + ssh + """ + def __init__(self, user, remote_host, cmd): + self.user = user + self.host = remote_host + self.cmd = cmd + self.client = paramiko.SSHClient() + self.client.load_system_host_keys() + self.client.connect(self.host, username=self.user) + + def run(self): + stdin, stdout, stderr = self.client.exec_command(self.cmd) + return (stdin, stdout, stderr) commit c0a56cbf52e0f5d4fa8862a59e5d5665ad71d80c Author: adam stokes <uzr(a)jak.rebo> Date: Mon Sep 28 12:00:00 2009 -0400 - updated spec diff --git a/cas.spec b/cas.spec index 462ae10..708bbcc 100644 --- a/cas.spec +++ b/cas.spec @@ -2,8 +2,8 @@ Name: cas Summary: Tool to analyze and configure core file environment -Version: 0.14 -Release: 10%{?dist} +Version: 0.15 +Release: 1%{?dist} Source0: https://fedorahosted.org/releases/c/a/cas/%{name}-%{version}.tar.gz License: GPLv3+ Group: Development/Libraries @@ -12,9 +12,9 @@ BuildArch: noarch Url: http://fedorahosted.org/cas BuildRequires: python-devel %if 0%{?rhel} -Requires: python-sqlite crash +Requires: python-sqlite crash PyYAML %else -Requires: crash +Requires: crash python-paramiko PyYAML %endif %description @@ -49,6 +49,10 @@ rm -rf ${RPM_BUILD_ROOT} %doc AUTHORS LICENSE README PKG-INFO doc/* %changelog +* Tue Sep 15 2009 Adam Stokes <ajs at redhat dot com> - 0.15-1 +- Require PyYAML, paramiko +- Rip out func code + * Tue May 5 2009 Adam Stokes <ajs at redhat dot com> - 0.14-8 - support for purging old data - documentation updated to reflect updated workflow and describe

1 0

cas cas.conf lib/cas
by Adam Stokes 28 Sep '09

28 Sep '09

cas | 33 +++++++++++---------------------- cas.conf | 5 +++++ lib/cas/core.py | 6 ++++-- 3 files changed, 20 insertions(+), 24 deletions(-) New commits: commit ee9864a931fe18c5a58942205d9cfd0551ad9002 Author: adam stokes <uzr(a)jak.rebo> Date: Mon Sep 28 11:33:16 2009 -0400 merge buffersize settings from stokes-devel branch diff --git a/cas b/cas index b6d20cc..f4e3f9d 100755 --- a/cas +++ b/cas @@ -38,6 +38,8 @@ if sys.version_info[:2] < (2,3): raise SystemExit("Python >= 2.3 required") # Configuration parsing of /etc/cas.conf +# TODO: rework config read to place all items +# into a key,value pair automatically config = ConfigParser.ConfigParser() config.read("/etc/cas.conf") WORKDIRECTORY = config.get("settings","workDirectory") @@ -54,6 +56,11 @@ CRASH_32=None if config.has_option("advanced", "crash_32"): CRASH_32=config.get("advanced", "crash_32") +# Do we have a buffersize +BUFFERSIZE=None +if config.has_option("advanced", "buffersize"): + BUFFERSIZE=config.get("advanced", "buffersize") + class CoreHandler(object): def __init__(self, filename, dst, logger): self.filename = filename @@ -115,7 +122,7 @@ class TimestampHandler(object): # dig through the buildstamp database and attempt to match it with the # one found in the core try: - coreTimestamp = self.tool.timestamp(self.corefile) + coreTimestamp = self.tool.timestamp(self.corefile, BUFFERSIZE) except CoreException, err: raise SystemExit(self.casLog.debug(err)) # query timestamp table, return tuple debuginfoRPM, and path to @@ -252,12 +259,9 @@ class CasApplication(object): self.casLog.debug("crash problem: err: %s, out: %s" % (err, out)) else: # The machine running cas isn't capable of processing this core, lets - # attempt with Func. Assuming Func is installed and a server database + # attempt with paramiko. Assuming paramiko is installed and a server database # is configured we attempt to process the core at another machine. try: - import func.overlord.client as fc - import func.jobthing as jobthing - from time import sleep self.casLog.info("Crash file built, locating suitable %s system for " \ "processing" % (debugKernelArch,)) serverList = self.db.getServers() @@ -269,22 +273,7 @@ class CasApplication(object): self.casLog.info("Machine %s found, processing " \ "crash output" % (casProcessMachine,)) cmd = "cd " + os.path.join(self.storagePath) + " && ./crash -i crash.in > crash.out" - client = fc.Overlord(casProcessMachine,async=True) - # test if client is reachable - isClientReachable = client.run("test", "ping", []) - if isClientReachable == 1: - job_id = client.command.run(cmd) - while True: - status = client.job_status(job_id) - (code, results) = status - if code == jobthing.JOB_ID_RUNNING: - self.casLog.status("task is running...") - elif code == jobthing.JOB_ID_PARTIAL: - self.casLog.status("task is running...") - elif code == jobthing.JOB_ID_FINISHED: - self.casLog.status("task complete; results = %s" % (results,)) - break - sleep(5) + # TODO: run command through paramiko else: self.casLog.info("No servers available for arch and current system not "\ "suitable for processing, please run cas-admin -h " \ @@ -294,7 +283,7 @@ class CasApplication(object): "more information")) except ImportError: self.casLog.info("Current running machine is not suitable for processing this core " \ - "and http://fedorahosted.org/func is not installed/configured properly.") + "and python-paramiko is not installed/configured properly.") self.casLog.info("Finishing job without processing the core, please find a suitable %s "\ "machine in order to view this core in crash." % (debugKernelArch,)) crashOutFile = os.path.join(self.storagePath,"modules") diff --git a/cas.conf b/cas.conf index 8df8ac8..1928139 100644 --- a/cas.conf +++ b/cas.conf @@ -54,3 +54,8 @@ autoPurge=no # rpm --relocate /usr/bin=/usr/local/i386 -ivh crash*i386.rpm # crash_32=/usr/local/i386/crash + +# Define a buffersize to be set when reading a core file +# Valid options are {number, None, or blank} +# If set to None or blank default will be ~515M +# buffersize = None diff --git a/lib/cas/core.py b/lib/cas/core.py index f4558d5..010ad6f 100644 --- a/lib/cas/core.py +++ b/lib/cas/core.py @@ -93,7 +93,7 @@ class CoreBase(object): return True return False - def timestamp(self, path): + def timestamp(self, path, blksize): """ captures fingerprint from core """ match='Linux\sversion.*20\d{1,2}|#1\s.*20\d{1,2}' @@ -102,7 +102,9 @@ class CoreBase(object): except IOError: return False fd.seek(0) - b = os.read(fd.fileno(),54000000) + if not blksize or blksize == "": + blksize = 540000000 + b = os.read(fd.fileno(),blksize) out = self.util.regexSearch(match, b) if out: return out

1 0

Branch 'stokes-devel' - cas cas.conf lib/cas
by Adam Stokes 28 Sep '09

28 Sep '09

cas | 9 ++++++++- cas.conf | 5 +++++ lib/cas/core.py | 6 ++++-- 3 files changed, 17 insertions(+), 3 deletions(-) New commits: commit c581b7f84420ef252f06e7c279be5a704aa9a6c0 Author: adam stokes <uzr(a)jak.rebo> Date: Mon Sep 28 11:27:47 2009 -0400 - add buffersize option diff --git a/cas b/cas index f69b68b..f4e3f9d 100755 --- a/cas +++ b/cas @@ -38,6 +38,8 @@ if sys.version_info[:2] < (2,3): raise SystemExit("Python >= 2.3 required") # Configuration parsing of /etc/cas.conf +# TODO: rework config read to place all items +# into a key,value pair automatically config = ConfigParser.ConfigParser() config.read("/etc/cas.conf") WORKDIRECTORY = config.get("settings","workDirectory") @@ -54,6 +56,11 @@ CRASH_32=None if config.has_option("advanced", "crash_32"): CRASH_32=config.get("advanced", "crash_32") +# Do we have a buffersize +BUFFERSIZE=None +if config.has_option("advanced", "buffersize"): + BUFFERSIZE=config.get("advanced", "buffersize") + class CoreHandler(object): def __init__(self, filename, dst, logger): self.filename = filename @@ -115,7 +122,7 @@ class TimestampHandler(object): # dig through the buildstamp database and attempt to match it with the # one found in the core try: - coreTimestamp = self.tool.timestamp(self.corefile) + coreTimestamp = self.tool.timestamp(self.corefile, BUFFERSIZE) except CoreException, err: raise SystemExit(self.casLog.debug(err)) # query timestamp table, return tuple debuginfoRPM, and path to diff --git a/cas.conf b/cas.conf index 8df8ac8..1928139 100644 --- a/cas.conf +++ b/cas.conf @@ -54,3 +54,8 @@ autoPurge=no # rpm --relocate /usr/bin=/usr/local/i386 -ivh crash*i386.rpm # crash_32=/usr/local/i386/crash + +# Define a buffersize to be set when reading a core file +# Valid options are {number, None, or blank} +# If set to None or blank default will be ~515M +# buffersize = None diff --git a/lib/cas/core.py b/lib/cas/core.py index f4558d5..010ad6f 100644 --- a/lib/cas/core.py +++ b/lib/cas/core.py @@ -93,7 +93,7 @@ class CoreBase(object): return True return False - def timestamp(self, path): + def timestamp(self, path, blksize): """ captures fingerprint from core """ match='Linux\sversion.*20\d{1,2}|#1\s.*20\d{1,2}' @@ -102,7 +102,9 @@ class CoreBase(object): except IOError: return False fd.seek(0) - b = os.read(fd.fileno(),54000000) + if not blksize or blksize == "": + blksize = 540000000 + b = os.read(fd.fileno(),blksize) out = self.util.regexSearch(match, b) if out: return out

1 0

Branch 'stokes-devel' - cas.spec doc/fixtures setup.py
by Adam Stokes 15 Sep '09

15 Sep '09

cas.spec | 8 ++++++-- doc/fixtures/servers.yml | 17 +++++++++++++++++ setup.py | 2 +- 3 files changed, 24 insertions(+), 3 deletions(-) New commits: commit 2733622948be561909957b7f99193375149706a9 Author: adam stokes <uzr(a)jak.rebo> Date: Tue Sep 15 14:40:14 2009 -0400 - setup pre-existing fixtures for ease of adding servers diff --git a/cas.spec b/cas.spec index 3cf34df..708bbcc 100644 --- a/cas.spec +++ b/cas.spec @@ -12,9 +12,9 @@ BuildArch: noarch Url: http://fedorahosted.org/cas BuildRequires: python-devel %if 0%{?rhel} -Requires: python-sqlite crash +Requires: python-sqlite crash PyYAML %else -Requires: crash python-paramiko +Requires: crash python-paramiko PyYAML %endif %description @@ -49,6 +49,10 @@ rm -rf ${RPM_BUILD_ROOT} %doc AUTHORS LICENSE README PKG-INFO doc/* %changelog +* Tue Sep 15 2009 Adam Stokes <ajs at redhat dot com> - 0.15-1 +- Require PyYAML, paramiko +- Rip out func code + * Tue May 5 2009 Adam Stokes <ajs at redhat dot com> - 0.14-8 - support for purging old data - documentation updated to reflect updated workflow and describe diff --git a/doc/fixtures/servers.yml b/doc/fixtures/servers.yml new file mode 100644 index 0000000..a8f4bb2 --- /dev/null +++ b/doc/fixtures/servers.yml @@ -0,0 +1,17 @@ +# Server listing in YAML +# Supported architectures: x86_64, i686, s390, s390x, ppc, ppc64, ia64 + +username: root +password: password +ssh_key: id_dsa + +server1.example.com: + architecture: x86_64 + username: testuser # overwrite master username + port: 22 +server2.example.com: + architecture: i686 + port: 22 +server3.example.com: + architecture: ppc64 + port: 22 diff --git a/setup.py b/setup.py index d5aba5b..16773ff 100644 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ for dirpath, dirnames, filenames in os.walk('snippets'): setup( name = 'cas', - version = '0.14', + version = '0.15', author = 'Adam Stokes', author_email = 'ajs(a)redhat.com', url = "http://fedorahosted.org/cas",

1 0

Branch 'stokes-devel' - cas cas-admin cas.spec
by Adam Stokes 15 Sep '09

15 Sep '09

cas | 24 +++--------------------- cas-admin | 31 +++++++------------------------ cas.spec | 6 +++--- 3 files changed, 13 insertions(+), 48 deletions(-) New commits: commit d91d764b5e51eb8789506e7f687398feae4e610d Author: adam stokes <uzr(a)jak.rebo> Date: Tue Sep 15 11:20:57 2009 -0400 - begin ripping out func code. diff --git a/cas b/cas index b6d20cc..f69b68b 100755 --- a/cas +++ b/cas @@ -252,12 +252,9 @@ class CasApplication(object): self.casLog.debug("crash problem: err: %s, out: %s" % (err, out)) else: # The machine running cas isn't capable of processing this core, lets - # attempt with Func. Assuming Func is installed and a server database + # attempt with paramiko. Assuming paramiko is installed and a server database # is configured we attempt to process the core at another machine. try: - import func.overlord.client as fc - import func.jobthing as jobthing - from time import sleep self.casLog.info("Crash file built, locating suitable %s system for " \ "processing" % (debugKernelArch,)) serverList = self.db.getServers() @@ -269,22 +266,7 @@ class CasApplication(object): self.casLog.info("Machine %s found, processing " \ "crash output" % (casProcessMachine,)) cmd = "cd " + os.path.join(self.storagePath) + " && ./crash -i crash.in > crash.out" - client = fc.Overlord(casProcessMachine,async=True) - # test if client is reachable - isClientReachable = client.run("test", "ping", []) - if isClientReachable == 1: - job_id = client.command.run(cmd) - while True: - status = client.job_status(job_id) - (code, results) = status - if code == jobthing.JOB_ID_RUNNING: - self.casLog.status("task is running...") - elif code == jobthing.JOB_ID_PARTIAL: - self.casLog.status("task is running...") - elif code == jobthing.JOB_ID_FINISHED: - self.casLog.status("task complete; results = %s" % (results,)) - break - sleep(5) + # TODO: run command through paramiko else: self.casLog.info("No servers available for arch and current system not "\ "suitable for processing, please run cas-admin -h " \ @@ -294,7 +276,7 @@ class CasApplication(object): "more information")) except ImportError: self.casLog.info("Current running machine is not suitable for processing this core " \ - "and http://fedorahosted.org/func is not installed/configured properly.") + "and python-paramiko is not installed/configured properly.") self.casLog.info("Finishing job without processing the core, please find a suitable %s "\ "machine in order to view this core in crash." % (debugKernelArch,)) crashOutFile = os.path.join(self.storagePath,"modules") diff --git a/cas-admin b/cas-admin index 2e575d8..c51ba15 100755 --- a/cas-admin +++ b/cas-admin @@ -113,32 +113,15 @@ class CasServerHandler(object): try: serverList = {} hostname_count = 0 - # Attempt to load func to automate detection of machine arch - import func.overlord.client as fc - parent_func = fc.Overlord("*") - minions = parent_func.minions - for i in minions: - scheme, netloc, path, query, frag = urlparse.urlsplit(i) - hostname, port = netloc.split(":") - client = fc.Overlord(hostname) - isClientReachable = client.run("test","ping", []) - if isClientReachable == 1: - # Returns as {"hostname":['sts','stdout','stderr']} - # what a pain - client_dict = client.command.run("/bin/uname -m") - # silly magic to get output from command module - # since we are doing one host at a time we just pull the first - # key from the dict and parse its return - sts, arch, err = client_dict[client_dict.keys()[0]] - if sts: - self.casLog.debug(err) - # clean up arch string - arch = arch.strip() - self.db.addServer(hostname, arch) - hostname_count = hostname_count + 1 + # Attempt to load paramiko to automate detection of machine arch + # import func.overlord.client as fc + # TODO: add interactive session for adding servers, and also + # add ability to script this + # self.db.addServer(hostname, arch) + # hostname_count = hostname_count + 1 self.casLog.info("Server database built with %d server(s) added." % (hostname_count,)) except ImportError: - raise SystemExit(self.casLog.debug("Please install func (http://fedorahosted.org/func) for " \ + raise SystemExit(self.casLog.debug("Please install python-paramiko for " \ "an automated machine arch population.\n")) return diff --git a/cas.spec b/cas.spec index 462ae10..3cf34df 100644 --- a/cas.spec +++ b/cas.spec @@ -2,8 +2,8 @@ Name: cas Summary: Tool to analyze and configure core file environment -Version: 0.14 -Release: 10%{?dist} +Version: 0.15 +Release: 1%{?dist} Source0: https://fedorahosted.org/releases/c/a/cas/%{name}-%{version}.tar.gz License: GPLv3+ Group: Development/Libraries @@ -14,7 +14,7 @@ BuildRequires: python-devel %if 0%{?rhel} Requires: python-sqlite crash %else -Requires: crash +Requires: crash python-paramiko %endif %description

1 0

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

2009

2008

cas September 2009