Source code for openmdao.util.fileutil

"""
Misc. file utility routines.
"""

import os
import stat
import sys
import shutil
import warnings
import itertools
import string
from hashlib import md5

from fnmatch import fnmatch
from os.path import isdir, join
from os.path import dirname, basename, exists, isfile, abspath


[docs]class DirContext(object): """Supports using the 'with' statement in place of try-finally for entering a directory, executing a block, then returning to the original directory. """ def __init__(self, destdir): self.destdir = destdir def __enter__(self): self.startdir = os.getcwd() # convert destdir to absolute at enter time instead of init time # so relative paths will be relative to the current context self.destdir = os.path.abspath(self.destdir) os.chdir(self.destdir) return self.destdir def __exit__(self, exc_type, exc_val, exc_tb): os.chdir(self.startdir)
[docs]def expand_path(path): return os.path.abspath(os.path.expandvars(os.path.expanduser(path)))
[docs]def cleanup(*fnames, **kwargs): """Delete the given files or directories if they exists.""" for fname in fnames: if os.path.isfile(fname): os.remove(fname) elif os.path.isdir(fname): shutil.rmtree(fname, **kwargs)
[docs]def find_in_dir_list(fname, dirlist, exts=('',)): """Search the given list of directories for the specified file. Return the absolute path of the file if found, or None otherwise. fname: str Base name of file. dirlist: list of str List of directory paths, relative or absolute. exts: tuple of str Tuple of extensions (including the '.') to apply to fname for loop, e.g., ('.exe','.bat'). """ for path in dirlist: for ext in exts: fpath = join(path, fname)+ext if isfile(fpath): return abspath(fpath) return None
[docs]def find_in_path(fname, pathvar=None, sep=os.pathsep, exts=('',)): """Search for a given file in all of the directories given in the pathvar string. Return the absolute path to the file if found, or None otherwise. fname: str Base name of file. pathvar: str String containing search paths. Defaults to $PATH. sep: str Delimiter used to separate paths within pathvar. exts: tuple of str Tuple of extensions (including the '.') to apply to fname for loop, e.g., ('.exe','.bat'). """ if pathvar is None: pathvar = os.environ['PATH'] return find_in_dir_list(fname, pathvar.split(sep), exts)
def _file_gen(dname, fmatch=bool, dmatch=None): """A generator returning files under the given directory, with optional file and directory filtering. fmatch: predicate funct A predicate function that returns True on a match. This is used to match files only. dmatch: predicate funct A predicate function that returns True on a match. This is used to match directories only. """ if dmatch is not None and not dmatch(dname): return for path, dirlist, filelist in os.walk(dname): if dmatch is not None: # prune directories to search newdl = [d for d in dirlist if dmatch(d)] if len(newdl) != len(dirlist): dirlist[:] = newdl # replace contents of dirlist to cause pruning for name in [f for f in filelist if fmatch(f)]: yield join(path, name) def _file_dir_gen(dname, fmatch=bool, dmatch=None): """A generator returning files and directories under the given directory, with optional file and directory filtering.. fmatch: predicate funct A predicate function that returns True on a match. This is used to match files only. dmatch: predicate funct A predicate function that returns True on a match. This is used to match directories only. """ if dmatch is not None and not dmatch(dname): return for path, dirlist, filelist in os.walk(dname): if dmatch is not None: # prune directories to search newdl = [d for d in dirlist if dmatch(d)] if len(newdl) != len(dirlist): dirlist[:] = newdl # replace contents of dirlist to cause pruning for name in [f for f in filelist if fmatch(f)]: yield join(path, name) for name in dirlist: yield join(path, name)
[docs]def find_files(start, match=None, exclude=None, showdirs=False, dirmatch=None, direxclude=None): """Return filenames (using a generator). start: str or list of str Starting directory or list of directories. match: str or predicate funct Either a string containing a glob pattern to match or a predicate function that returns True on a match. This is used to match files only. exclude: str or predicate funct Either a string containing a glob pattern to exclude or a predicate function that returns True to exclude. This is used to exclude files only. showdirs: bool If True, return names of files AND directories. dirmatch: str or predicate funct Either a string containing a glob pattern to match or a predicate function that returns True on a match. This is used to match directories only. direxclude: str or predicate funct Either a string containing a glob pattern to exclude or a predicate function that returns True to exclude. This is used to exclude directories only. Walks all subdirectories below each specified starting directory, subject to directory filtering. """ startdirs = [start] if isinstance(start, basestring) else start if len(startdirs) == 0: return iter([]) gen = _file_dir_gen if showdirs else _file_gen if match is None: matcher = bool elif isinstance(match, basestring): matcher = lambda name: fnmatch(name, match) else: matcher = match if dirmatch is None: dmatcher = bool elif isinstance(dirmatch, basestring): dmatcher = lambda name: fnmatch(name, dirmatch) else: dmatcher = dirmatch if isinstance(exclude, basestring): fmatch = lambda name: matcher(name) and not fnmatch(name, exclude) elif exclude is not None: fmatch = lambda name: matcher(name) and not exclude(name) else: fmatch = matcher if isinstance(direxclude, basestring): dmatch = lambda name: dmatcher(name) and not fnmatch(name, direxclude) elif direxclude is not None: dmatch = lambda name: dmatcher(name) and not direxclude(name) else: dmatch = dmatcher iters = [gen(d, fmatch=fmatch, dmatch=dmatch) for d in startdirs] if len(iters) > 1: return itertools.chain(*iters) else: return iters[0]
[docs]def find_up(name, path=None): """Search upward from the starting path (or the current directory) until the given file or directory is found. The given name is assumed to be a basename, not a path. Returns the absolute path of the file or directory if found, or None otherwise. name: str Base name of the file or directory being searched for. path: str (optional) Starting directory. If not supplied, current directory is used. """ if not path: path = os.getcwd() if not exists(path): return None while path: if exists(join(path, name)): return abspath(join(path, name)) else: pth = path path = dirname(path) if path == pth: return None return None
[docs]def get_module_path(fpath): """Given a module filename, return its full Python name including enclosing packages. (based on existence of ``__init__.py`` files) """ if basename(fpath).startswith('__init__.'): pnames = [] else: pnames = [os.path.splitext(basename(fpath))[0]] path = os.path.dirname(os.path.abspath(fpath)) while os.path.isfile(os.path.join(path, '__init__.py')): path, pname = os.path.split(path) pnames.append(pname) return '.'.join(pnames[::-1])
[docs]def find_module(name, path=None, py=True): """Return the pathname of the Python file corresponding to the given module name, or None if it can't be found. If path is set, search in path for the file; otherwise, search in sys.path. If py is True, the file must be an uncompiled Python (.py) file. """ if path is None: path = sys.path nameparts = name.split('.') endings = [os.path.join(*nameparts)] endings.append(os.path.join(endings[0], '__init__.py')) endings[0] += '.py' if not py: endings.append(endings[0]+'c') endings.append(endings[0]+'o') endings.append(endings[0]+'d') for entry in path: for ending in endings: f = os.path.join(entry, ending) if os.path.isfile(f): return f return None
[docs]def get_ancestor_dir(path, num_levels=1): """Return the name of the directory that is 'num_levels' levels above the specified path. If num_levels is larger than the number of members in the path, then the root directory name will be returned. """ for i in range(num_levels): path = os.path.dirname(path) return path
[docs]def copy(src, dest): """Copy a file or directory.""" if isfile(src): shutil.copy(src, dest) elif isdir(src): shutil.copytree(src, dest)
[docs]def build_directory(dct, force=False, topdir='.'): """Create a directory structure based on the contents of a nested dict. The directory is created in the specified top directory or in the current working directory if one isn't specified. If a file being created already exists, a warning will be issued and the file will not be changed if force is False. If force is True, the file will be overwritten. The structure of the dict is as follows: if the value at a key is a dict, then that key is used to create a directory. Otherwise, the key is used to create a file, and the value stored at that key is written to the file. All keys must be relative names or a RuntimeError will be raised. """ #TODO: if a value stored in the dict is a callable, then call it and store # its return value as the contents of the file startdir = os.getcwd() topdir = os.path.abspath(topdir) try: for key, val in dct.items(): os.chdir(topdir) if os.path.isabs(key): raise RuntimeError("build_directory: key (%s) is not a relative name" % key) if isinstance(val, dict): # it's a dict, so this is a directory if not os.path.exists(key): os.makedirs(key) os.chdir(key) build_directory(val, force) else: # assume a string value. Use that value to create a file if os.path.exists(key) and force is False: warnings.warn("File '%s' already exists and will not be overwritten." % key, Warning) else: dname = os.path.dirname(key) if dname and not os.path.isdir(dname): os.makedirs(dname) with open(key, 'w') as f: f.write(val) finally: os.chdir(startdir)
[docs]def get_cfg_file(): """Attempts to get the /OpenMDAO-Framework/config/testhosts.cfg first; then gets the ~/.openmdao/testhosts.cfg next.""" bindir = os.path.dirname(sys.executable) branchdir = os.path.dirname(os.path.dirname(bindir)) cfgdir = os.path.join(branchdir, 'config') cfgfile = os.path.join(cfgdir, 'testhosts.cfg') altcfg = '~/.openmdao/testhosts.cfg' if os.path.isfile(cfgfile): return cfgfile else: return altcfg
[docs]def clean_filename(name): ''' Removes special characters from a proposed filename and replaces them with underscores. To be safe, we allow only a limited set of chars.''' valid_chars = "-_.()%s%s" % (string.ascii_letters, string.digits) return ''.join(c if c in valid_chars else '_' for c in name)
[docs]def is_dev_build(): return basename(get_ancestor_dir(sys.executable, 2)) == 'devenv'
[docs]def file_md5(fpath): """Return the MD5 digest for the given file""" f = open(fpath, 'rb') try: m = md5() while True: s = f.read(4096) if not s: break m.update(s) return m.hexdigest() finally: f.close()
[docs]def onerror(func, path, exc_info): """ Error handler for ``shutil.rmtree``. If the error is due to an access error (read only file), it attempts to add write permission and then retries. If the error is for another reason, it re-raises the error. Usage : ``shutil.rmtree(path, onerror=onerror)`` """ if not os.access(path, os.W_OK): # Is the error an access error ? os.chmod(path, stat.S_IWUSR) func(path) else: raise
OpenMDAO Home