--- /dev/null
+"""File System Proxy.
+
+Provide an OS-neutral view on a file system, locally or remotely.
+The functionality is geared towards implementing some sort of
+rdist-like utility between a Mac and a UNIX system.
+
+The module defines three classes:
+
+FSProxyLocal -- used for local access
+FSProxyServer -- used on the server side of remote access
+FSProxyClient -- used on the client side of remote access
+
+The remote classes are instantiated with an IP address and an optional
+verbosity flag.
+"""
+
+import server
+import client
+import md5
+import os
+import fnmatch
+from stat import *
+import time
+import fnmatch
+
+if os.name == 'mac':
+ import macfs
+ maxnamelen = 31
+else:
+ macfs = None
+ maxnamelen = 255
+
+skipnames = (os.curdir, os.pardir)
+
+
+class FSProxyLocal:
+
+ def __init__(self):
+ self._dirstack = []
+ self._ignore = ['*.pyc'] + self._readignore()
+
+ def _close(self):
+ while self._dirstack:
+ self.back()
+
+ def _readignore(self):
+ file = self._hide('ignore')
+ try:
+ f = open(file)
+ except IOError:
+ file = self._hide('synctree.ignorefiles')
+ try:
+ f = open(file)
+ except IOError:
+ return []
+ ignore = []
+ while 1:
+ line = f.readline()
+ if not line: break
+ if line[-1] == '\n': line = line[:-1]
+ ignore.append(line)
+ f.close()
+ return ignore
+
+ def _hidden(self, name):
+ if os.name == 'mac':
+ return name[0] == '(' and name[-1] == ')'
+ else:
+ return name[0] == '.'
+
+ def _hide(self, name):
+ if os.name == 'mac':
+ return '(%s)' % name
+ else:
+ return '.%s' % name
+
+ def visible(self, name):
+ if len(name) > maxnamelen: return 0
+ if name[-1] == '~': return 0
+ if name in skipnames: return 0
+ if self._hidden(name): return 0
+ head, tail = os.path.split(name)
+ if head or not tail: return 0
+ if macfs:
+ if os.path.exists(name) and not os.path.isdir(name):
+ try:
+ fs = macfs.FSSpec(name)
+ c, t = fs.GetCreatorType()
+ if t != 'TEXT': return 0
+ except macfs.error, msg:
+ print "***", name, msg
+ return 0
+ else:
+ if os.path.islink(name): return 0
+ if '\0' in open(name, 'rb').read(512): return 0
+ for ign in self._ignore:
+ if fnmatch.fnmatch(name, ign): return 0
+ return 1
+
+ def check(self, name):
+ if not self.visible(name):
+ raise os.error, "protected name %s" % repr(name)
+
+ def checkfile(self, name):
+ self.check(name)
+ if not os.path.isfile(name):
+ raise os.error, "not a plain file %s" % repr(name)
+
+ def pwd(self):
+ return os.getcwd()
+
+ def cd(self, name):
+ self.check(name)
+ save = os.getcwd(), self._ignore
+ os.chdir(name)
+ self._dirstack.append(save)
+ self._ignore = self._ignore + self._readignore()
+
+ def back(self):
+ if not self._dirstack:
+ raise os.error, "empty directory stack"
+ dir, ignore = self._dirstack[-1]
+ os.chdir(dir)
+ del self._dirstack[-1]
+ self._ignore = ignore
+
+ def _filter(self, files, pat = None):
+ if pat:
+ def keep(name, pat = pat):
+ return fnmatch.fnmatch(name, pat)
+ files = filter(keep, files)
+ files = filter(self.visible, files)
+ files.sort()
+ return files
+
+ def list(self, pat = None):
+ files = os.listdir(os.curdir)
+ return self._filter(files, pat)
+
+ def listfiles(self, pat = None):
+ files = os.listdir(os.curdir)
+ files = filter(os.path.isfile, files)
+ return self._filter(files, pat)
+
+ def listsubdirs(self, pat = None):
+ files = os.listdir(os.curdir)
+ files = filter(os.path.isdir, files)
+ return self._filter(files, pat)
+
+ def exists(self, name):
+ return self.visible(name) and os.path.exists(name)
+
+ def isdir(self, name):
+ return self.visible(name) and os.path.isdir(name)
+
+ def islink(self, name):
+ return self.visible(name) and os.path.islink(name)
+
+ def isfile(self, name):
+ return self.visible(name) and os.path.isfile(name)
+
+ def sum(self, name):
+ self.checkfile(name)
+ BUFFERSIZE = 1024*8
+ f = open(name)
+ sum = md5.new()
+ while 1:
+ buffer = f.read(BUFFERSIZE)
+ if not buffer:
+ break
+ sum.update(buffer)
+ return sum.digest()
+
+ def size(self, name):
+ self.checkfile(name)
+ return os.stat(name)[ST_SIZE]
+
+ def mtime(self, name):
+ self.checkfile(name)
+ return time.localtime(os.stat(name)[ST_MTIME])
+
+ def stat(self, name):
+ self.checkfile(name)
+ size = os.stat(name)[ST_SIZE]
+ mtime = time.localtime(os.stat(name)[ST_MTIME])
+ return size, mtime
+
+ def info(self, name):
+ sum = self.sum(name)
+ size = os.stat(name)[ST_SIZE]
+ mtime = time.localtime(os.stat(name)[ST_MTIME])
+ return sum, size, mtime
+
+ def _list(self, function, list):
+ if list is None:
+ list = self.listfiles()
+ res = []
+ for name in list:
+ try:
+ res.append((name, function(name)))
+ except (os.error, IOError):
+ res.append((name, None))
+ return res
+
+ def sumlist(self, list = None):
+ return self._list(self.sum, list)
+
+ def statlist(self, list = None):
+ return self._list(self.stat, list)
+
+ def mtimelist(self, list = None):
+ return self._list(self.mtime, list)
+
+ def sizelist(self, list = None):
+ return self._list(self.size, list)
+
+ def infolist(self, list = None):
+ return self._list(self.info, list)
+
+ def _dict(self, function, list):
+ if list is None:
+ list = self.listfiles()
+ dict = {}
+ for name in list:
+ try:
+ dict[name] = function(name)
+ except (os.error, IOError):
+ pass
+ return dict
+
+ def sumdict(self, list = None):
+ return self.dict(self.sum, list)
+
+ def sizedict(self, list = None):
+ return self.dict(self.size, list)
+
+ def mtimedict(self, list = None):
+ return self.dict(self.mtime, list)
+
+ def statdict(self, list = None):
+ return self.dict(self.stat, list)
+
+ def infodict(self, list = None):
+ return self._dict(self.info, list)
+
+ def read(self, name, offset = 0, length = -1):
+ self.checkfile(name)
+ f = open(name)
+ f.seek(offset)
+ if length == 0:
+ data = ''
+ elif length < 0:
+ data = f.read()
+ else:
+ data = f.read(length)
+ f.close()
+ return data
+
+ def create(self, name):
+ self.check(name)
+ if os.path.exists(name):
+ self.checkfile(name)
+ bname = name + '~'
+ try:
+ os.unlink(bname)
+ except os.error:
+ pass
+ os.rename(name, bname)
+ f = open(name, 'w')
+ f.close()
+
+ def write(self, name, data, offset = 0):
+ self.checkfile(name)
+ f = open(name, 'r+')
+ f.seek(offset)
+ f.write(data)
+ f.close()
+
+ def mkdir(self, name):
+ self.check(name)
+ os.mkdir(name, 0777)
+
+ def rmdir(self, name):
+ self.check(name)
+ os.rmdir(name)
+
+
+class FSProxyServer(FSProxyLocal, server.Server):
+
+ def __init__(self, address, verbose = server.VERBOSE):
+ FSProxyLocal.__init__(self)
+ server.Server.__init__(self, address, verbose)
+
+ def _close(self):
+ server.Server._close(self)
+ FSProxyLocal._close(self)
+
+ def _serve(self):
+ server.Server._serve(self)
+ # Retreat into start directory
+ while self._dirstack: self.back()
+
+
+class FSProxyClient(client.Client):
+
+ def __init__(self, address, verbose = client.VERBOSE):
+ client.Client.__init__(self, address, verbose)
+
+
+def test():
+ import string
+ import sys
+ if sys.argv[1:]:
+ port = string.atoi(sys.argv[1])
+ else:
+ port = 4127
+ proxy = FSProxyServer(('', port))
+ proxy._serverloop()
+
+
+if __name__ == '__main__':
+ test()
--- /dev/null
+"""RPC Client module."""
+
+import sys
+import socket
+import pickle
+import __builtin__
+import os
+
+
+# Default verbosity (0 = silent, 1 = print connections, 2 = print requests too)
+VERBOSE = 1
+
+
+class Client:
+
+ """RPC Client class. No need to derive a class -- it's fully generic."""
+
+ def __init__(self, address, verbose = VERBOSE):
+ if type(address) == type(0):
+ address = ('', address)
+ self._address = address
+ self._verbose = verbose
+ if self._verbose: print "Connecting to %s ..." % repr(address)
+ self._socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ self._socket.connect(address)
+ if self._verbose: print "Connected."
+ self._lastid = 0 # Last id for which a reply has been received
+ self._nextid = 1 # Id of next request
+ self._replies = {} # Unprocessed replies
+ self._rf = self._socket.makefile('r')
+ self._wf = self._socket.makefile('w')
+ self._methods = self._call('.methods')
+
+ def __del__(self):
+ self._close()
+
+ def _close(self):
+ if self._rf: self._rf.close()
+ self._rf = None
+ if self._wf: self._wf.close()
+ self._wf = None
+ if self._socket: self._socket.close()
+ self._socket = None
+
+ def __getattr__(self, name):
+ if name in self._methods:
+ method = _stub(self, name)
+ setattr(self, name, method) # XXX circular reference
+ return method
+ raise AttributeError, name
+
+ def _setverbose(self, verbose):
+ self._verbose = verbose
+
+ def _call(self, name, *args):
+ return self._vcall(name, args)
+
+ def _vcall(self, name, args):
+ return self._recv(self._vsend(name, args))
+
+ def _send(self, name, *args):
+ return self._vsend(name, args)
+
+ def _send_noreply(self, name, *args):
+ return self._vsend(name, args, 0)
+
+ def _vsend_noreply(self, name, args):
+ return self._vsend(name, args, 0)
+
+ def _vsend(self, name, args, wantreply = 1):
+ id = self._nextid
+ self._nextid = id+1
+ if not wantreply: id = -id
+ request = (name, args, id)
+ if self._verbose > 1: print "sending request: %s" % repr(request)
+ wp = pickle.Pickler(self._wf)
+ wp.dump(request)
+ return id
+
+ def _recv(self, id):
+ exception, value, rid = self._vrecv(id)
+ if rid != id:
+ raise RuntimeError, "request/reply id mismatch: %d/%d" % (id, rid)
+ if exception is None:
+ return value
+ x = exception
+ if hasattr(__builtin__, exception):
+ x = getattr(__builtin__, exception)
+ elif exception in ('posix.error', 'mac.error'):
+ x = os.error
+ if x == exception:
+ exception = x
+ raise exception, value
+
+ def _vrecv(self, id):
+ self._flush()
+ if self._replies.has_key(id):
+ if self._verbose > 1: print "retrieving previous reply, id = %d" % id
+ reply = self._replies[id]
+ del self._replies[id]
+ return reply
+ aid = abs(id)
+ while 1:
+ if self._verbose > 1: print "waiting for reply, id = %d" % id
+ rp = pickle.Unpickler(self._rf)
+ reply = rp.load()
+ del rp
+ if self._verbose > 1: print "got reply: %s" % repr(reply)
+ rid = reply[2]
+ arid = abs(rid)
+ if arid == aid:
+ if self._verbose > 1: print "got it"
+ return reply
+ self._replies[rid] = reply
+ if arid > aid:
+ if self._verbose > 1: print "got higher id, assume all ok"
+ return (None, None, id)
+
+ def _flush(self):
+ self._wf.flush()
+
+
+class _stub:
+
+ """Helper class for Client -- each instance serves as a method of the client."""
+
+ def __init__(self, client, name):
+ self._client = client
+ self._name = name
+
+ def __call__(self, *args):
+ return self._client._vcall(self._name, args)
+
--- /dev/null
+"""Compare local and remote dictionaries and transfer differing files -- like rdist."""
+
+import sys
+from repr import repr
+import FSProxy
+import time
+import os
+
+def main():
+ pwd = os.getcwd()
+ s = raw_input("chdir [%s] " % pwd)
+ if s:
+ os.chdir(s)
+ pwd = os.getcwd()
+ host = ask("host", 'voorn.cwi.nl')
+ port = 4127
+ verbose = 1
+ mode = ''
+ print """\
+Mode should be a string of characters, indicating what to do with differences.
+r - read different files to local file system
+w - write different files to remote file system
+c - create new files, either remote or local
+d - delete disappearing files, either remote or local
+"""
+ s = raw_input("mode [%s] " % mode)
+ if s: mode = s
+ address = (host, port)
+ t1 = time.time()
+ local = FSProxy.FSProxyLocal()
+ remote = FSProxy.FSProxyClient(address, verbose)
+ compare(local, remote, mode)
+ remote._close()
+ local._close()
+ t2 = time.time()
+ dt = t2-t1
+ mins, secs = divmod(dt, 60)
+ print mins, "minutes and", secs, "seconds"
+ raw_input("[Return to exit] ")
+
+def ask(prompt, default):
+ s = raw_input("%s [%s] " % (prompt, default))
+ return s or default
+
+def askint(prompt, default):
+ s = raw_input("%s [%s] " % (prompt, str(default)))
+ if s: return string.atoi(s)
+ return default
+
+def compare(local, remote, mode):
+ print
+ print "PWD =", `os.getcwd()`
+ sums_id = remote._send('sumlist')
+ subdirs_id = remote._send('listsubdirs')
+ remote._flush()
+ print "calculating local sums ..."
+ lsumdict = {}
+ for name, info in local.sumlist():
+ lsumdict[name] = info
+ print "getting remote sums ..."
+ sums = remote._recv(sums_id)
+ print "got", len(sums)
+ rsumdict = {}
+ for name, rsum in sums:
+ rsumdict[name] = rsum
+ if not lsumdict.has_key(name):
+ print `name`, "only remote"
+ if 'r' in mode and 'c' in mode:
+ recvfile(local, remote, name)
+ else:
+ lsum = lsumdict[name]
+ if lsum != rsum:
+ print `name`,
+ rmtime = remote.mtime(name)
+ lmtime = local.mtime(name)
+ if rmtime > lmtime:
+ print "remote newer",
+ if 'r' in mode:
+ recvfile(local, remote, name)
+ elif lmtime > rmtime:
+ print "local newer",
+ if 'w' in mode:
+ sendfile(local, remote, name)
+ else:
+ print "same mtime but different sum?!?!",
+ print
+ for name in lsumdict.keys():
+ if not rsumdict.keys():
+ print `name`, "only locally",
+ fl()
+ if 'w' in mode and 'c' in mode:
+ sendfile(local, remote, name)
+ elif 'r' in mode and 'd' in mode:
+ os.unlink(name)
+ print "removed."
+ print
+ print "gettin subdirs ..."
+ subdirs = remote._recv(subdirs_id)
+ common = []
+ for name in subdirs:
+ if local.isdir(name):
+ print "Common subdirectory", repr(name)
+ common.append(name)
+ else:
+ print "Remote subdirectory", repr(name), "not found locally"
+ lsubdirs = local.listsubdirs()
+ for name in lsubdirs:
+ if name not in subdirs:
+ print "Local subdirectory", repr(name), "not found remotely"
+ for name in common:
+ print "Entering subdirectory", repr(name)
+ local.cd(name)
+ remote.cd(name)
+ compare(local, remote, mode)
+ remote.back()
+ local.back()
+
+def sendfile(local, remote, name):
+ try:
+ remote.create(name)
+ except (IOError, os.error), msg:
+ print "cannot create:", msg
+ return
+
+ print "sending ...",
+ fl()
+
+ data = open(name).read()
+
+ t1 = time.time()
+
+ remote._send_noreply('write', name, data)
+ remote._flush()
+
+ t2 = time.time()
+
+ dt = t2-t1
+ print len(data), "bytes in", t2-t1, "seconds",
+ if dt:
+ print "i.e.", len(data)/dt, "bytes/sec",
+ print
+
+def recvfile(local, remote, name):
+ try:
+ local.create(name)
+ except (IOError, os.error), msg:
+ print "cannot create:", msg
+ return
+
+ print "receiving ...",
+ fl()
+
+ f = open(name, 'w')
+ t1 = time.time()
+
+ length = 4*1024
+ offset = 0
+ id = remote._send('read', name, offset, length)
+ remote._flush()
+ while 1:
+ newoffset = offset + length
+ newid = remote._send('read', name, newoffset, length)
+ data = remote._recv(id)
+ id = newid
+ if not data: break
+ f.seek(offset)
+ f.write(data)
+ offset = newoffset
+ size = f.tell()
+
+ t2 = time.time()
+ f.close()
+
+ dt = t2-t1
+ print size, "bytes in", dt, "seconds",
+ if dt:
+ print "i.e.", int(size/dt), "bytes/sec",
+ print
+ remote._recv(id) # ignored
+
+def fl():
+ sys.stdout.flush()
+
+if __name__ == '__main__':
+ main()
--- /dev/null
+"""RPC Server module."""
+
+import sys
+import socket
+import pickle
+from fnmatch import fnmatch
+from repr import repr
+
+
+# Default verbosity (0 = silent, 1 = print connections, 2 = print requests too)
+VERBOSE = 1
+
+
+class Server:
+
+ """RPC Server class. Derive a class to implement a particular service."""
+
+ def __init__(self, address, verbose = VERBOSE):
+ if type(address) == type(0):
+ address = ('', address)
+ self._address = address
+ self._verbose = verbose
+ self._socket = None
+ self._socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ self._socket.bind(address)
+ self._socket.listen(1)
+ self._listening = 1
+
+ def _setverbose(self, verbose):
+ self._verbose = verbose
+
+ def __del__(self):
+ self._close()
+
+ def _close(self):
+ self._listening = 0
+ if self._socket:
+ self._socket.close()
+ self._socket = None
+
+ def _serverloop(self):
+ while self._listening:
+ self._serve()
+
+ def _serve(self):
+ if self._verbose: print "Wait for connection ..."
+ conn, address = self._socket.accept()
+ if self._verbose: print "Accepted connection from %s" % repr(address)
+ if not self._verify(conn, address):
+ print "*** Connection from %s refused" % repr(address)
+ conn.close()
+ return
+ rf = conn.makefile('r')
+ wf = conn.makefile('w')
+ ok = 1
+ while ok:
+ wf.flush()
+ if self._verbose > 1: print "Wait for next request ..."
+ ok = self._dorequest(rf, wf)
+
+ _valid = ['192.16.201.*', '192.16.197.*']
+
+ def _verify(self, conn, address):
+ host, port = address
+ for pat in self._valid:
+ if fnmatch(host, pat): return 1
+ return 0
+
+ def _dorequest(self, rf, wf):
+ rp = pickle.Unpickler(rf)
+ try:
+ request = rp.load()
+ except EOFError:
+ return 0
+ if self._verbose > 1: print "Got request: %s" % repr(request)
+ try:
+ methodname, args, id = request
+ if '.' in methodname:
+ reply = (None, self._special(methodname, args), id)
+ elif methodname[0] == '_':
+ raise NameError, "illegal method name %s" % repr(methodname)
+ else:
+ method = getattr(self, methodname)
+ reply = (None, apply(method, args), id)
+ except:
+ reply = (sys.exc_type, sys.exc_value, id)
+ if id < 0 and reply[:2] == (None, None):
+ if self._verbose > 1: print "Suppress reply"
+ return 1
+ if self._verbose > 1: print "Send reply: %s" % repr(reply)
+ wp = pickle.Pickler(wf)
+ wp.dump(reply)
+ return 1
+
+ def _special(self, methodname, args):
+ if methodname == '.methods':
+ if not hasattr(self, '_methods'):
+ self._methods = tuple(self._listmethods())
+ return self._methods
+ raise NameError, "unrecognized special method name %s" % repr(methodname)
+
+ def _listmethods(self, cl=None):
+ if not cl: cl = self.__class__
+ names = cl.__dict__.keys()
+ names = filter(lambda x: x[0] != '_', names)
+ names.sort()
+ for base in cl.__bases__:
+ basenames = self._listmethods(base)
+ basenames = filter(lambda x, names=names: x not in names, basenames)
+ names[len(names):] = basenames
+ return names
--- /dev/null
+import time
+import FSProxy
+
+def main():
+ t1 = time.time()
+ #proxy = FSProxy.FSProxyClient(('voorn.cwi.nl', 4127))
+ proxy = FSProxy.FSProxyLocal()
+ sumtree(proxy)
+ proxy._close()
+ t2 = time.time()
+ print t2-t1, "seconds"
+ raw_input("[Return to exit] ")
+
+def sumtree(proxy):
+ print "PWD =", proxy.pwd()
+ files = proxy.listfiles()
+ proxy.infolist(files)
+ subdirs = proxy.listsubdirs()
+ for name in subdirs:
+ proxy.cd(name)
+ sumtree(proxy)
+ proxy.back()
+
+main()
projects / thirdparty / Python / cpython.git / commitdiff
