vdsm-patches July 2011

vdsm-patches@lists.fedorahosted.org

13 participants
183 discussions

[NEW PATCH] Change getChain() to return dictionary with chain itself and its template. (via gerrit-bot)
by Igor Lvovsky 24 May '12

24 May '12

New patch submitted by Igor Lvovsky (ilvovsky(a)redhat.com) You can review this change at: http://gerrit.usersys.redhat.com/692 commit 3a09f874889cb372ee84b04a234f7b653458e26f Author: Igor Lvovsky <ilvovsky(a)redhat.com> Date: Tue Jul 12 15:18:45 2011 +0300 Change getChain() to return dictionary with chain itself and its template. This change will avoid unneeded produceVolume and improve behaviour of different flows. Change-Id: Ife46d592171aedea087e0d6462b735cd7e0d75e0 diff --git a/vdsm/storage/image.py b/vdsm/storage/image.py index 1c70a34..102ea6c 100644 --- a/vdsm/storage/image.py +++ b/vdsm/storage/image.py @@ -104,7 +104,7 @@ class Image: # 1. Remove template's image: Create 'fake' template instead of deleted one # 2. Remove regular image: Remove parent-'fake' template if nobody need it already try: - pvol = self.getTemplate(sdUUID=sdUUID, imgUUID=imgUUID) + pvol = self.getChain(sdUUID, imgUUID)['template'] # 1. If we required to delete template's image that have VMs # based on it, we should create similar 'fake' template instead if pvol: @@ -269,17 +269,16 @@ class Image: return newImgUUID - def __chainSizeCalc(self, sdUUID, imgUUID, volUUID, size): + def __chainSizeCalc(self, sdUUID, imgUUID, size): """ Compute an estimate of the whole chain size using the sum of the actual size of the chain's volumes """ - chain = self.getChain(sdUUID, imgUUID, volUUID) + chainDict = self.getChain(sdUUID, imgUUID) newsize = 0 - template = chain[0].getParentVolume() - if template: - newsize = template.getVolumeSize() - for vol in chain: + if chainDict['template']: + newsize = chainDict['template'].getVolumeSize() + for vol in chainDict['chain']: newsize += vol.getVolumeSize() if newsize > size: newsize = size @@ -301,12 +300,14 @@ class Image: newsize = int(newsize * 1.1) # allocate %10 more for cow metadata return newsize - def getChain(self, sdUUID, imgUUID, volUUID=None): + def getChain(self, sdUUID, imgUUID): """ - Return the chain of volumes of image as a sorted list - (not including a shared base (template) if any) + Return the dictionary with chain of volumes of image as a sorted list + (not including a shared base (template)) and the template if any """ - chain = [] + # 'chain' - chain of volumes of image as a sorted list + # 'template' - shared base (template) if any + chainDict = {'chain':[], 'template':None} # Find all volumes of image volclass = SDF.produce(sdUUID).getVolumeClass() uuidlist = volclass.getImageVolumes(self.repoPath, sdUUID, imgUUID) @@ -316,14 +317,13 @@ class Image: srcVol = volclass(self.repoPath, sdUUID, imgUUID, uuidlist[0]) # For template image include only one volume (template itself) if len(uuidlist) == 1 and srcVol.isShared(): - return [srcVol] + return {'chain':[srcVol], 'template':srcVol} # find the leaf for vol in uuidlist: srcVol = volclass(self.repoPath, sdUUID, imgUUID, vol) if srcVol.isLeaf(): - if not volUUID or volUUID == srcVol.volUUID: - break + break srcVol = None if not srcVol: @@ -332,38 +332,25 @@ class Image: # Build up the sorted (parent->child) chain while not srcVol.isShared(): - chain.insert(0, srcVol) + chainDict['chain'].insert(0, srcVol) if srcVol.getParent() == volume.BLANK_UUID: break srcVol = srcVol.getParentVolume() - self.log.info("sdUUID=%s imgUUID=%s chain=%s ", sdUUID, imgUUID, str(chain)) - return chain + if srcVol.isShared(): + chainDict['template'] = srcVol - def getTemplate(self, sdUUID, imgUUID): - """ - Return template of the image - """ - tmpl = None - # Find all volumes of image (excluding template) - chain = self.getChain(sdUUID, imgUUID) - # check if the chain is build above a template, or it is a standalone - pvol = chain[0].getParentVolume() - if pvol: - tmpl = pvol - elif chain[0].isShared(): - tmpl = chain[0] - - return tmpl + self.log.info("sdUUID=%s imgUUID=%s chainDict=%s ", sdUUID, imgUUID, str(chainDict)) + return chainDict def validate(self, srcSdUUID, dstSdUUID, imgUUID, op=MOVE_OP): """ Validate template on destination domain """ - # Find all volumes of source image - chain = self.getChain(srcSdUUID, imgUUID) - leafVol = chain[-1] srcDom = SDF.produce(srcSdUUID) + # Find all volumes of source image + chainDict = self.getChain(srcSdUUID, imgUUID) + leafVol = chainDict['chain'][-1] # Avoid move template's image if there is a VM based on it (except 'Backup' domain) if op == MOVE_OP and leafVol.isShared() and not srcDom.isBackup(): chList = leafVol.getAllChildrenList(self.repoPath, srcSdUUID, imgUUID, leafVol.volUUID) @@ -371,18 +358,16 @@ class Image: raise se.MoveTemplateImageError(imgUUID) # check if the chain is build above a template, or it is a standalone - pvol = chain[0].getParentVolume() + pvol = chainDict['template'] if pvol: # this is a shared template based chain - if not pvol.isShared(): - raise se.ImageIsNotLegalChain("Base image parent vol %s is not shared" % pvol.volUUID) pimg = pvol.getImage() # pimg == template image try: volclass = SDF.produce(dstSdUUID).getVolumeClass() # Validate that the destination template exists and accessible volclass(self.repoPath, dstSdUUID, pimg, pvol.volUUID) - except se.StorageException, e: - self.log.error("Unexpected error", exc_info=True) - raise se.CouldNotValideTemplateOnTargetDomain("Template %s Destination domain %s: %s" % (pimg, dstSdUUID, str(e))) + except se.StorageException: + self.log.error("Cannot validate template %s on target domain %s", pimg, dstSdUUID, exc_info=True) + raise se.CouldNotValideTemplateOnTargetDomain() def __templateRelink(self, destDom, imgUUID, volUUID): """ @@ -473,20 +458,15 @@ class Image: def _createTargetImage(self, destDom, srcSdUUID, imgUUID): # Before actual data copying we need perform several operation # such as: create all volumes, create fake template if needed, ... - try: - # Find all volumes of source image - srcChain = self.getChain(srcSdUUID, imgUUID) - except se.StorageException: - self.log.error("Unexpected error", exc_info=True) - raise - except Exception, e: - self.log.error("Unexpected error", exc_info=True) - raise se.SourceImageActionError(imgUUID, srcSdUUID, str(e)) + + # Find all volumes of source image + srcChainDict = self.getChain(srcSdUUID, imgUUID) + srcChain = srcChainDict['chain'] fakeTemplate = False pimg = volume.BLANK_UUID # standalone chain # check if the chain is build above a template, or it is a standalone - pvol = srcChain[0].getParentVolume() + pvol = srcChainDict['template'] if pvol: # find out parent volume parameters volParams = pvol.getVolumeParams() @@ -686,11 +666,9 @@ class Image: """ if not self.isLegal(sdUUID, imgUUID): raise se.ImageIsNotLegalChain(imgUUID) - chain = self.getChain(sdUUID, imgUUID) - # check if the chain is build above a template, or it is a standalone - pvol = chain[0].getParentVolume() - if pvol: - if not pvol.isLegal() or pvol.isFake(): + chainDict = self.getChain(sdUUID, imgUUID) + if chainDict['template']: + if not chainDict['template'].isLegal() or chainDict['template'].isFake(): raise se.ImageIsNotLegalChain(imgUUID) def copy(self, sdUUID, vmUUID, srcImgUUID, srcVolUUID, dstImgUUID, dstVolUUID, @@ -735,8 +713,7 @@ class Image: # using the sum of the actual size of the chain's volumes if volParams['volFormat'] != volume.COW_FORMAT or volParams['prealloc'] != volume.SPARSE_VOL: raise se.IncorrectFormat(self) - volParams['apparentsize'] = self.__chainSizeCalc(sdUUID, srcImgUUID, - srcVolUUID, volParams['size']) + volParams['apparentsize'] = self.__chainSizeCalc(sdUUID, srcImgUUID, volParams['size']) # Find out dest volume parameters if preallocate in [volume.PREALLOCATED_VOL, volume.SPARSE_VOL]: diff --git a/vdsm/storage/resourceFactories.py b/vdsm/storage/resourceFactories.py index ce3ec70..07df0f0 100644 --- a/vdsm/storage/resourceFactories.py +++ b/vdsm/storage/resourceFactories.py @@ -106,20 +106,18 @@ class ImageResourceFactory(rm.SimpleResourceFactory): # Get the list of the volumes repoPath = os.path.join(self.storage_repository, dom.getPools()[0]) try: - chain = image.Image(repoPath).getChain(sdUUID=self.sdUUID, imgUUID=resourceName) + chainDict = image.Image(repoPath).getChain(sdUUID=self.sdUUID, imgUUID=resourceName) + chain = chainDict['chain'] except se.ImageDoesNotExistInSD: log.debug("Image %s does not exist in domain %s", resourceName, self.sdUUID) return [] - # check if the chain is build above a template, or it is a standalone - pvol = chain[0].getParentVolume() - if pvol: - template = pvol.volUUID - elif chain[0].isShared(): - # Image of template itself, - # with no other volumes in chain - template = chain[0].volUUID - del chain[:] + # Image of template itself, + # with no other volumes in chain + if chainDict['template']: + template = chainDict['template'].volUUID + if chainDict['template'] in chain: + del chain[:] volUUIDChain = [vol.volUUID for vol in chain] volUUIDChain.sort()

4 5

[NEW PATCH] Removing references to sdf.py. (via gerrit-bot)
by ewarszaw＠redhat.com 22 Oct '11

22 Oct '11

New patch submitted by Eduardo Warszawski (ewarszaw(a)redhat.com) You can review this change at: http://gerrit.usersys.redhat.com/644 commit dd4884ae7cbf5d090c8e405dbd85705c72969b65 Author: Eduardo Warszawski <ewarszaw(a)redhat.com> Date: Fri Jul 1 10:11:47 2011 +0300 Removing references to sdf.py. Change-Id: I19d178e3b37fa119dc8a44815bb5055535c95241 diff --git a/vdsm/storage/blockVolume.py b/vdsm/storage/blockVolume.py index ae411be..107ae69 100644 --- a/vdsm/storage/blockVolume.py +++ b/vdsm/storage/blockVolume.py @@ -23,7 +23,7 @@ import task import lvm import resourceManager as rm from threadLocal import vars -from sdf import StorageDomainFactory as SDF +from sdc import StorageDomainFactory as SDF from resourceFactories import LVM_ACTIVATION_NAMESPACE import fileUtils diff --git a/vdsm/storage/fileVolume.py b/vdsm/storage/fileVolume.py index 9ed54ff..fc4008f 100644 --- a/vdsm/storage/fileVolume.py +++ b/vdsm/storage/fileVolume.py @@ -13,7 +13,7 @@ import uuid from config import config import storage_exception as se -from sdf import StorageDomainFactory as SDF +from sdc import StorageDomainFactory as SDF import volume import image import sd diff --git a/vdsm/storage/hsm.py b/vdsm/storage/hsm.py index 296e923..f139b40 100644 --- a/vdsm/storage/hsm.py +++ b/vdsm/storage/hsm.py @@ -28,7 +28,7 @@ import spm import lvm import fileUtils import multipath -from sdf import StorageDomainFactory as SDF +from sdc import StorageDomainFactory as SDF import volume import iscsi import misc @@ -1346,7 +1346,7 @@ class HSM: domains = domList.keys() else: #getSharedLock(connectionsResource...) - domains = SDF.getAllUUIDs() + domains = SDF.getUUIDs() for sdUUID in domains[:]: try: diff --git a/vdsm/storage/image.py b/vdsm/storage/image.py index 40a82c9..2e7c302 100644 --- a/vdsm/storage/image.py +++ b/vdsm/storage/image.py @@ -14,7 +14,7 @@ import threading import uuid import volume -from sdf import StorageDomainFactory as SDF +from sdc import StorageDomainFactory as SDF import sd import misc import fileUtils diff --git a/vdsm/storage/resourceFactories.py b/vdsm/storage/resourceFactories.py index ce3ec70..42cd72a 100644 --- a/vdsm/storage/resourceFactories.py +++ b/vdsm/storage/resourceFactories.py @@ -13,7 +13,7 @@ import logging import lvm import resourceManager as rm import storage_exception as se -from sdf import StorageDomainFactory as SDF +from sdc import StorageDomainFactory as SDF import sd import image diff --git a/vdsm/storage/sdc.py b/vdsm/storage/sdc.py index 883f7d2..8db7b11 100644 --- a/vdsm/storage/sdc.py +++ b/vdsm/storage/sdc.py @@ -13,6 +13,7 @@ for keeping storage related data that is expensive to harvest, but needed often import logging import threading import weakref +from config import config import multipath import lvm @@ -22,6 +23,7 @@ import storage_exception as se # Default cache age until forcibly refreshed DEFAULT_REFRESH_INTERVAL = 300 + class StorageDomainCache: """ Storage Domain List keeps track of all the storage domains accessible by the @@ -58,7 +60,7 @@ class StorageDomainCache: del self.__weakCache[sdUUID] - def lookup(self, sdUUID): + def produce(self, sdUUID): dom = self._getDomainFromCache(sdUUID) if dom: return dom @@ -121,3 +123,8 @@ class StorageDomainCache: def manuallyRemoveDomain(self, sdUUID): with self._syncroot: del self.__cache[sdUUID] + + +storage_repository = config.get('irs', 'repository') +StorageDomainFactory = StorageDomainCache(storage_repository) + diff --git a/vdsm/storage/sp.py b/vdsm/storage/sp.py index c377ee4..39d1135 100644 --- a/vdsm/storage/sp.py +++ b/vdsm/storage/sp.py @@ -24,7 +24,7 @@ import misc from misc import Event import fileUtils from config import config -from sdf import StorageDomainFactory as SDF +from sdc import StorageDomainFactory as SDF import storage_exception as se from persistentDict import DictValidator diff --git a/vdsm/storage/spm.py b/vdsm/storage/spm.py index 085d03b..dd603f2 100644 --- a/vdsm/storage/spm.py +++ b/vdsm/storage/spm.py @@ -29,7 +29,7 @@ import sd import hsm import blockSD import image -from sdf import StorageDomainFactory as SDF +from sdc import StorageDomainFactory as SDF import volume import misc import logging diff --git a/vdsm/storage/volume.py b/vdsm/storage/volume.py index 123ed37..6fb4dfb 100644 --- a/vdsm/storage/volume.py +++ b/vdsm/storage/volume.py @@ -16,7 +16,7 @@ import signal import constants import storage_exception as se import sd -from sdf import StorageDomainFactory as SDF +from sdc import StorageDomainFactory as SDF import misc import fileUtils import task

4 7

[NEW PATCH] BZ#717658 - Simplify SDF.produce() (via gerrit-bot)
by ewarszaw＠redhat.com 22 Oct '11

22 Oct '11

New patch submitted by Eduardo Warszawski (ewarszaw(a)redhat.com) You can review this change at: http://gerrit.usersys.redhat.com/643 commit f30ec1f48c499c1f74de19be88744a7fa70f5b51 Author: Eduardo Warszawski <ewarszaw(a)redhat.com> Date: Fri Jul 1 08:04:43 2011 +0300 BZ#717658 - Simplify SDF.produce() produce() calls lookup(), lookup() calls (blockSD, localFsSD, nfsSD).findDomain(), findDomain() calls __init__(), __init__() calls findDomainPath(), findDomainPath() will return or raise StorageDomainDoesNotExist! Never None, never a false value, no need for this check. Change-Id: I442c368a1266855b324eab96e4a7b02634984834 diff --git a/vdsm/storage/sdc.py b/vdsm/storage/sdc.py index 1a8a3fa..883f7d2 100644 --- a/vdsm/storage/sdc.py +++ b/vdsm/storage/sdc.py @@ -73,6 +73,7 @@ class StorageDomainCache: self._cleanStaleWeakrefs() + #_findDomain will raise StorageDomainDoesNotExist if sdUUID is not found in storage. dom = self._findDomain(sdUUID) self.__cache[sdUUID] = dom self.__weakCache[sdUUID] = weakref.ref(dom) diff --git a/vdsm/storage/sdf.py b/vdsm/storage/sdf.py index 58fdea0..9d09b3b 100644 --- a/vdsm/storage/sdf.py +++ b/vdsm/storage/sdf.py @@ -11,7 +11,6 @@ from config import config import logging import sdc -import storage_exception as se class StorageDomainFactory: @@ -34,10 +33,7 @@ class StorageDomainFactory: Produce a new Storage domain """ - newSD = cls.__sdc.lookup(sdUUID) - if not newSD: - raise se.StorageDomainDoesNotExist(sdUUID) - return newSD + return cls.__sdc.lookup(sdUUID) @classmethod

2 5

[NEW PATCH] BZ#717658 - Remove SDF.create() (via gerrit-bot)
by ewarszaw＠redhat.com 22 Oct '11

22 Oct '11

New patch submitted by Eduardo Warszawski (ewarszaw(a)redhat.com) You can review this change at: http://gerrit.usersys.redhat.com/642 commit d15b149a4e5adc8d5763e4ff2aa31f4d050637a6 Author: Eduardo Warszawski <ewarszaw(a)redhat.com> Date: Fri Jul 1 06:58:20 2011 +0300 BZ#717658 - Remove SDF.create() Change-Id: I31b64def7aea87d4d3a94950abc8e96fbe3c4b8a diff --git a/vdsm/storage/hsm.py b/vdsm/storage/hsm.py index c4fa249..296e923 100644 --- a/vdsm/storage/hsm.py +++ b/vdsm/storage/hsm.py @@ -22,6 +22,8 @@ from itertools import imap import sp import sd import blockSD +import nfsSD +import localFsSD import spm import lvm import fileUtils @@ -1171,8 +1173,15 @@ class HSM: #getSharedLock(connectionsResource...) #getExclusiveLock(sdUUID...) - SDF.create(sdUUID=sdUUID, storageType=storageType, domainName=domainName, - domClass=domClass, typeSpecificArg=typeSpecificArg, version=domVersion) + if storageType in sd.BLOCK_DOMAIN_TYPES: + newSD = blockSD.BlockStorageDomain.create(sdUUID, domainName, domClass, typeSpecificArg, storageType, domVersion) + elif storageType == sd.NFS_DOMAIN: + newSD = nfsSD.NfsStorageDomain.create(sdUUID, domainName, domClass, typeSpecificArg, storageType, domVersion) + elif storageType == sd.LOCALFS_DOMAIN: + newSD = localFsSD.LocalFsStorageDomain.create(sdUUID, domainName, domClass, typeSpecificArg, storageType, domVersion) + else: + raise se.StorageDomainTypeError(storageType) + SDF.manuallyAddDomain(newSD) def public_validateStorageDomain(self, sdUUID, options = None): diff --git a/vdsm/storage/sdf.py b/vdsm/storage/sdf.py index 5e0577d..58fdea0 100644 --- a/vdsm/storage/sdf.py +++ b/vdsm/storage/sdf.py @@ -11,7 +11,6 @@ from config import config import logging import sdc -import sd import storage_exception as se @@ -20,6 +19,11 @@ class StorageDomainFactory: storage_repository = config.get('irs', 'repository') __sdc = sdc.StorageDomainCache(storage_repository) + #WARNING! The parameters of the following two methods are not symmetric. + @classmethod + def manuallyAddDomain(cls, sd): + cls.__sdc.manuallyAddDomain(sd) + @classmethod def manuallyRemoveDomain(cls, sdUUID): cls.__sdc.manuallyRemoveDomain(sdUUID) @@ -37,38 +41,6 @@ class StorageDomainFactory: @classmethod - def create(cls, sdUUID, storageType, domainName, domClass, typeSpecificArg, version): - """ - Create a new Storage domain - """ - import nfsSD - import localFsSD - import blockSD - - newSD = None - if storageType in [sd.NFS_DOMAIN]: - newSD = nfsSD.NfsStorageDomain.create(sdUUID=sdUUID, - domainName=domainName, domClass=domClass, - remotePath=typeSpecificArg, storageType=storageType, - version=version) - elif storageType in [sd.LOCALFS_DOMAIN]: - newSD = localFsSD.LocalFsStorageDomain.create(sdUUID=sdUUID, - domainName=domainName, domClass=domClass, - remotePath=typeSpecificArg, storageType=storageType, - version=version) - elif storageType in [sd.ISCSI_DOMAIN, sd.FCP_DOMAIN]: - newSD = blockSD.BlockStorageDomain.create(sdUUID=sdUUID, - domainName=domainName, domClass=domClass, - vgUUID=typeSpecificArg, storageType=storageType, - version=version) - else: - raise se.StorageDomainTypeError(storageType) - - cls.__sdc.manuallyAddDomain(newSD) - return newSD - - - @classmethod def getAllUUIDs(cls): return cls.__sdc.getUUIDs()

2 5

[NEW PATCH] BZ#717658 - Simplify recycle function. (via gerrit-bot)
by ewarszaw＠redhat.com 22 Oct '11

22 Oct '11

New patch submitted by Eduardo Warszawski (ewarszaw(a)redhat.com) You can review this change at: http://gerrit.usersys.redhat.com/641 commit 4ef2f06aca02bc48f63c3b5f7d48ae83e191b3ba Author: Eduardo Warszawski <ewarszaw(a)redhat.com> Date: Thu Jun 30 16:18:50 2011 +0300 BZ#717658 - Simplify recycle function. Change-Id: I9cf3106f0fb5d5cd067907d48ff02fe50493a66c diff --git a/vdsm/storage/hsm.py b/vdsm/storage/hsm.py index 29206f5..c4fa249 100644 --- a/vdsm/storage/hsm.py +++ b/vdsm/storage/hsm.py @@ -1190,6 +1190,15 @@ class HSM: return SDF.produce(sdUUID=sdUUID).validate() + #TODO: Remove this function when formatStorageDomain() is removed. + def _recycle(self, sd): + try: + SDF.manuallyRemoveDomain(sd.sdUUID) + except KeyError: + self.log.warn("Storage domain %s doesn't exist in cache. Trying recycle leftovers ...", sd.sdUUID) + + sd.format(sd.sdUUID) + def public_formatStorageDomain(self, sdUUID, autoDetach = False, options = None): """ Formats a detached storage domain. @@ -1198,6 +1207,7 @@ class HSM: This removes all data from the storage domain. :param sdUUID: The UUID for the storage domain you want to format. + :param autoDetach: DEPRECATED :type sdUUID: UUID :param options: ? @@ -1214,22 +1224,17 @@ class HSM: raise se.CannotFormatStorageDomainInConnectedPool(sdUUID) # For domains that attached to disconnected pool, format domain if 'autoDetach' flag set - if not misc.parseBool(autoDetach): + sd = SDF.produce(sdUUID=sdUUID) + try: + #TODO: autoDetach is True + if not misc.parseBool(autoDetach) and sd.getPools(): + raise se.CannotFormatAttachedStorageDomain(sdUUID) # Allow format also for broken domain - try: - if len(SDF.produce(sdUUID=sdUUID).getPools()) > 0: - raise se.CannotFormatAttachedStorageDomain(sdUUID) - - except (se.StorageDomainMetadataNotFound, se.MetaDataGeneralError, se.MiscFileReadException, - se.MiscBlockReadException, se.MiscBlockReadIncomplete), e: - self.log.warn("Domain %s has problem with metadata. Continue formating... (%s)", sdUUID, str(e)) - except se.CannotFormatAttachedStorageDomain: - raise - except Exception: - self.log.warn("Domain %s can't be formated", sdUUID, exc_info=True) - raise se.StorageDomainFormatError(sdUUID) + except (se.StorageDomainMetadataNotFound, se.MetaDataGeneralError, se.MiscFileReadException, + se.MiscBlockReadException, se.MiscBlockReadIncomplete), e: + self.log.warn("Domain %s has problem with metadata. Continue formating... (%s)", sdUUID, str(e)) - SDF.recycle(sdUUID=sdUUID) + self._recycle(sd) def public_setStorageDomainDescription(self, sdUUID, description, options = None): diff --git a/vdsm/storage/sdf.py b/vdsm/storage/sdf.py index 16faa3e..5e0577d 100644 --- a/vdsm/storage/sdf.py +++ b/vdsm/storage/sdf.py @@ -20,6 +20,9 @@ class StorageDomainFactory: storage_repository = config.get('irs', 'repository') __sdc = sdc.StorageDomainCache(storage_repository) + @classmethod + def manuallyRemoveDomain(cls, sdUUID): + cls.__sdc.manuallyRemoveDomain(sdUUID) @classmethod def produce(cls, sdUUID): @@ -66,33 +69,6 @@ class StorageDomainFactory: @classmethod - def recycle(cls, sdUUID): - """ - Cleanly destroys the domain - """ - import nfsSD - import localFsSD - import blockSD - - try: - cls.__sdc.manuallyRemoveDomain(sdUUID) - except Exception: - cls.log.warn("Storage domain %s doesn't exist. Trying recycle leftovers ...", sdUUID) - - for domClass in (blockSD.BlockStorageDomain, nfsSD.NfsStorageDomain, localFsSD.LocalFsStorageDomain): - try: - domClass.findDomainPath(sdUUID) - except (se.StorageDomainDoesNotExist): - pass - except Exception: - cls.log.error("Can't find out domain %s", sdUUID, exc_info=True) - else: - return domClass.format(sdUUID) - - raise se.StorageDomainTypeError(sdUUID) - - - @classmethod def getAllUUIDs(cls): return cls.__sdc.getUUIDs()

2 5

[NEW PATCH] BZ#717658 - Change format signature. (via gerrit-bot)
by ewarszaw＠redhat.com 22 Oct '11

22 Oct '11

New patch submitted by Eduardo Warszawski (ewarszaw(a)redhat.com) You can review this change at: http://gerrit.usersys.redhat.com/640 commit a609d198d38137374db72a7ac3fc416c36ad64d3 Author: Eduardo Warszawski <ewarszaw(a)redhat.com> Date: Thu Jun 30 16:07:58 2011 +0300 BZ#717658 - Change format signature. Change-Id: Iea1e5735563087195181c625e57b8ba8e75d3441 diff --git a/vdsm/storage/blockSD.py b/vdsm/storage/blockSD.py index 214e8a4..5c8f8e2 100644 --- a/vdsm/storage/blockSD.py +++ b/vdsm/storage/blockSD.py @@ -658,11 +658,12 @@ class BlockStorageDomain(sd.StorageDomain): fileUtils.cleanupdir(os.path.join("/dev", self.sdUUID)) @classmethod - def format(cls, sdUUID, domaindir): + def format(cls, sdUUID): """Format detached storage domain. This removes all data from the storage domain. """ # Remove the directory tree + domaindir = cls.findDomainPath(sdUUID) fileUtils.cleanupdir(domaindir, ignoreErrors = True) # Remove special metadata and service volumes # Remove all volumes LV if exists diff --git a/vdsm/storage/fileSD.py b/vdsm/storage/fileSD.py index 8b40a59..023c197 100644 --- a/vdsm/storage/fileSD.py +++ b/vdsm/storage/fileSD.py @@ -195,12 +195,13 @@ class FileStorageDomain(sd.StorageDomain): return imgList @classmethod - def format(cls, sdUUID, domaindir): + def format(cls, sdUUID): """ Format detached storage domain. This removes all data from the storage domain. """ cls.log.info("Formating domain %s", sdUUID) + domaindir = cls.findDomainPath(sdUUID) processPoolDict[sdUUID].fileUtils.cleanupdir(domaindir, ignoreErrors = False) return True diff --git a/vdsm/storage/sd.py b/vdsm/storage/sd.py index 325da5d..579b282 100644 --- a/vdsm/storage/sd.py +++ b/vdsm/storage/sd.py @@ -552,14 +552,6 @@ class StorageDomain: if self.isBackup(): self.unmountMaster() - - def format(self): - """ - Format detached storage domain. - This removes all data from the storage domain. - """ - pass - def getAllImages(self): """ Fetch the list of the Image UUIDs diff --git a/vdsm/storage/sdf.py b/vdsm/storage/sdf.py index da8335c..16faa3e 100644 --- a/vdsm/storage/sdf.py +++ b/vdsm/storage/sdf.py @@ -81,13 +81,13 @@ class StorageDomainFactory: for domClass in (blockSD.BlockStorageDomain, nfsSD.NfsStorageDomain, localFsSD.LocalFsStorageDomain): try: - domaindir = domClass.findDomainPath(sdUUID) + domClass.findDomainPath(sdUUID) except (se.StorageDomainDoesNotExist): pass except Exception: cls.log.error("Can't find out domain %s", sdUUID, exc_info=True) else: - return domClass.format(sdUUID, domaindir) + return domClass.format(sdUUID) raise se.StorageDomainTypeError(sdUUID)

3 6

[NEW PATCH] BZ#717658 - Remove SDF.invalidateStorage. (via gerrit-bot)
by ewarszaw＠redhat.com 18 Sep '11

18 Sep '11

New patch submitted by Eduardo Warszawski (ewarszaw(a)redhat.com) You can review this change at: http://gerrit.usersys.redhat.com/638 commit 3862cfc24502f249e2d1a298cde2ab8a7ef6d581 Author: Eduardo Warszawski <ewarszaw(a)redhat.com> Date: Thu Jun 30 06:06:00 2011 +0300 BZ#717658 - Remove SDF.invalidateStorage. Change-Id: I13a6c7a91e372d6c53ffaddfbe527d6fe1a5c6d7 diff --git a/vdsm/storage/sdc.py b/vdsm/storage/sdc.py index a3ea775..0ed34e5 100644 --- a/vdsm/storage/sdc.py +++ b/vdsm/storage/sdc.py @@ -37,7 +37,7 @@ class StorageDomainCache: self.storageStale = True - def invalidateStorage(self): + def _invalidateStorage(self): self.storageStale = True @misc.samplingmethod @@ -117,7 +117,7 @@ class StorageDomainCache: return uuids def refresh(self): - self.invalidateStorage() + self._invalidateStorage() self.__cache.clear() def manuallyAddDomain(self, dom): diff --git a/vdsm/storage/sdf.py b/vdsm/storage/sdf.py index b5ec3e3..da8335c 100644 --- a/vdsm/storage/sdf.py +++ b/vdsm/storage/sdf.py @@ -103,11 +103,6 @@ class StorageDomainFactory: @classmethod - def invalidateStorage(cls): - cls.__sdc.invalidateStorage() - - - @classmethod def refreshStorage(cls): cls.__sdc.refreshStorage()

1 1

[NEW PATCH] BZ#717658 - Forward one step the reloading of block domains. (via gerrit-bot)
by ewarszaw＠redhat.com 18 Sep '11

18 Sep '11

New patch submitted by Eduardo Warszawski (ewarszaw(a)redhat.com) You can review this change at: http://gerrit.usersys.redhat.com/637 commit 0f27a3fcc8ecbe2e7f3925d0251a05c033b01d64 Author: Eduardo Warszawski <ewarszaw(a)redhat.com> Date: Thu Jun 30 05:40:58 2011 +0300 BZ#717658 - Forward one step the reloading of block domains. SDF.invalidateStorage() will trigger SDF.refreshStorage when SDF.lookup, but lvm reload will came only after. Change-Id: I035060894720518d70a0951ffc0d0c14a98793d2 diff --git a/vdsm/storage/hsm.py b/vdsm/storage/hsm.py index ab5a59d..29206f5 100644 --- a/vdsm/storage/hsm.py +++ b/vdsm/storage/hsm.py @@ -1055,7 +1055,7 @@ class HSM: statusList = storage_connection.StorageServerConnection().connect(domType=domType, conList=conList) # Connecting new device may change the visible storage domain list # so invalidate caches - SDF.invalidateStorage() + SDF.refreshStorage() return dict(statuslist=statusList)

1 1

[NEW PATCH] Remove check for incoming migration in _highWrite (via gerrit-bot)
by Federico Simoncelli 15 Sep '11

15 Sep '11

New patch submitted by Federico Simoncelli (fsimonce(a)redhat.com) You can review this change at: http://gerrit.usersys.redhat.com/706 commit e6a428e85ab03c1e7126eef65872c70e16020f07 Author: Federico Simoncelli <fsimonce(a)redhat.com> Date: Wed Jul 13 16:25:04 2011 +0000 Remove check for incoming migration in _highWrite Change-Id: I1c27d429b1c610a5a2d6d1c5b342e0564673c315 diff --git a/vdsm/libvirtvm.py b/vdsm/libvirtvm.py index 40567a5..bf27692 100644 --- a/vdsm/libvirtvm.py +++ b/vdsm/libvirtvm.py @@ -52,9 +52,6 @@ class VmStatsThread(utils.AdvancedStatsThread): self.sampleCpu, self.sampleDisk, self.sampleNet) def _highWrite(self): - if self._vm._incomingMigrationPending(): - return - for vmDrive in self._vm._drives: if vmDrive.blockDev and vmDrive.format == 'cow': capacity, alloc, physical = \

5 8

[NEW PATCH] BZ#????? - one process pool to rule them all (via gerrit-bot)
by smizrahi＠redhat.com 06 Sep '11

06 Sep '11

New patch submitted by Saggi Mizrahi (smizrahi(a)redhat.com) You can review this change at: http://gerrit.usersys.redhat.com/744 commit 6978ada2d810ac73e2fda4c67af52e388bdc1c75 Author: Saggi Mizrahi <smizrahi(a)redhat.com> Date: Mon Jul 25 16:18:36 2011 +0300 BZ#????? - one process pool to rule them all Instead of having a process pool per domain use one pool but limit the amount of workers a domain can hold at any given time. Change-Id: Ia69071a049761b271a982c600cf0520a782c7eac diff --git a/vdsm/config.py b/vdsm/config.py index 488aa7b..dbed034 100644 --- a/vdsm/config.py +++ b/vdsm/config.py @@ -146,9 +146,10 @@ config.set('irs', 'gc_blocker_force_collect_interval', '60') config.set('irs', 'maximum_domains_in_pool', '100') # Process Pool Configuration -config.set('irs', 'process_pool_size', '20') +config.set('irs', 'process_pool_size', '100') config.set('irs', 'process_pool_timeout', '60') config.set('irs', 'process_pool_grace_period', '2') +config.set("irs", "process_pool_max_slots_per_domain", '10') ##################################################################### config.add_section('addresses') diff --git a/vdsm/storage/outOfProcess.py b/vdsm/storage/outOfProcess.py index 48a556a..e1a4de7 100644 --- a/vdsm/storage/outOfProcess.py +++ b/vdsm/storage/outOfProcess.py @@ -2,15 +2,18 @@ import os as mod_os import glob as mod_glob import types from config import config +import threading +from functools import wraps from fileUtils import open_ex import fileUtils as mod_fileUtils -from processPool import ProcessPool +from processPool import ProcessPool, NoFreeHelpersError MAX_HELPERS = config.getint("irs", "process_pool_size") GRACE_PERIOD = config.getint("irs", "process_pool_grace_period") DEFAULT_TIMEOUT = config.getint("irs", "process_pool_timeout") +HELPERS_PER_DOMAIN = config.getint("irs", "process_pool_max_slots_per_domain") _globalPool = ProcessPool(MAX_HELPERS, GRACE_PERIOD, DEFAULT_TIMEOUT) @@ -67,6 +70,32 @@ setattr(os, 'path', _ModuleWrapper(mod_os.path)) fileUtils = _ModuleWrapper(mod_fileUtils) +class ProcessPoolLimiter(object): + def __init__(self, procPool, limit): + self._procPool = procPool + self._limit = limit + self._lock = threading.Lock() + self._counter = 0 + + def wrapFunction(self, func): + @wraps(func) + def wrapper(*args, **kwds): + return self.runExternally(func, *args, **kwds) + return wrapper + + def runExternally(self, *args, **kwargs): + with self._lock: + if self._counter >= self._limit: + raise NoFreeHelpersError("You reached the process limit") + + self._counter += 1 + try: + return self._procPool.runExternally(*args, **kwargs) + finally: + with self._lock: + self._counter -= 1 + + class OopWrapper(object): def __init__(self, procPool): self._processPool = procPool diff --git a/vdsm/storage/processPool.py b/vdsm/storage/processPool.py index f8ef4af..98a32e2 100644 --- a/vdsm/storage/processPool.py +++ b/vdsm/storage/processPool.py @@ -23,7 +23,7 @@ class ProcessPool(object): self._maxSubProcess = maxSubProcess self._gracePeriod = gracePeriod self.timeout = timeout - self._helperPool = [None] * self._maxSubProcess + self._helperPool = [Helper() for i in range(self._maxSubProcess)] self._lockPool = [Lock() for i in range(self._maxSubProcess)] self._closed = False diff --git a/vdsm/storage/sd.py b/vdsm/storage/sd.py index 4422487..344d578 100644 --- a/vdsm/storage/sd.py +++ b/vdsm/storage/sd.py @@ -224,6 +224,7 @@ class ProcessPoolDict(dict): def __init__(self): dict.__init__(self) self._lock = threading.Lock() + self._pool = None def __getitem__(self, key): try: @@ -235,8 +236,14 @@ class ProcessPoolDict(dict): return dict.__getitem__(self, key) def _createProcessPool(self, key): - _domainPool = ProcessPool(oop.MAX_HELPERS, oop.GRACE_PERIOD, oop.DEFAULT_TIMEOUT) - return oop.OopWrapper(_domainPool) + # I initialize the pool dict on first call + # because it's created on import and generating + # hundreds of subprocess on import is not recommended + if self._pool is None: + with self._lock: + if self._pool is None: + self._pool = ProcessPool(oop.MAX_HELPERS, oop.GRACE_PERIOD, oop.DEFAULT_TIMEOUT) + return oop.OopWrapper(oop.ProcessPoolLimiter(self._pool, oop.HELPERS_PER_DOMAIN)) # Dictionary for process pools per sdUUID processPoolDict = ProcessPoolDict()

3 7

Jump to page:

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

vdsm-patches July 2011