Totally new "freeze" program.

diff --git a/Tools/freeze/findmodules.py b/Tools/freeze/findmodules.py
new file mode 100644 (file)
index 0000000..177f1fd
--- /dev/null
+++ b/Tools/freeze/findmodules.py
@@ -0,0 +1,128 @@
+# Determine the names and filenames of the modules imported by a
+# script, recursively.  This is done by scanning for lines containing
+# import statements.  (The scanning has only superficial knowledge of
+# Python syntax and no knowledge of semantics, so in theory the result
+# may be incorrect -- however this is quite unlikely if you don't
+# intentionally obscure your Python code.)
+
+import os
+import regex
+import string
+import sys
+
+
+# Top-level interface.
+# First argument is the main program (script).
+# Second optional argument is list of modules to be searched as well.
+
+def findmodules(scriptfile, modules = [], path = sys.path):
+       todo = {}
+       todo['__main__'] = scriptfile
+       for name in modules:
+               mod = os.path.basename(name)
+               if mod[-3:] == '.py': mod = mod[:-3]
+               todo[mod] = name
+       done = closure(todo)
+       return done
+
+
+# Compute the closure of scanfile() and findmodule().
+# Return a dictionary mapping module names to filenames.
+# Writes to stderr if a file can't be or read.
+
+def closure(todo):
+       done = {}
+       while todo:
+               newtodo = {}
+               for modname in todo.keys():
+                       if not done.has_key(modname):
+                               filename = todo[modname]
+                               if filename is None:
+                                       filename = findmodule(modname)
+                               done[modname] = filename
+                               if filename in ('<builtin>', '<unknown>'):
+                                       continue
+                               try:
+                                       modules = scanfile(filename)
+                               except IOError, msg:
+                                       sys.stderr.write("%s: %s\n" %
+                                                        (filename, str(msg)))
+                                       continue
+                               for m in modules:
+                                       if not done.has_key(m):
+                                               newtodo[m] = None
+               todo = newtodo
+       return done
+
+
+# Scan a file looking for import statements.
+# Return list of module names.
+# Can raise IOError.
+
+importstr = '\(^\|:\)[ \t]*import[ \t]+\([a-zA-Z0-9_, \t]+\)'
+fromstr   = '\(^\|:\)[ \t]*from[ \t]+\([a-zA-Z0-9_]+\)[ \t]+import[ \t]+'
+isimport = regex.compile(importstr)
+isfrom = regex.compile(fromstr)
+
+def scanfile(filename):
+       allmodules = {}
+       f = open(filename, 'r')
+       try:
+               while 1:
+                       line = f.readline()
+                       if not line: break # EOF
+                       while line[-2:] == '\\\n': # Continuation line
+                               line = line[:-2] + ' '
+                               line = line + f.readline()
+                       if isimport.search(line) >= 0:
+                               rawmodules = isimport.group(2)
+                               modules = string.splitfields(rawmodules, ',')
+                               for i in range(len(modules)):
+                                       modules[i] = string.strip(modules[i])
+                       elif isfrom.search(line) >= 0:
+                               modules = [isfrom.group(2)]
+                       else:
+                               continue
+                       for mod in modules:
+                               allmodules[mod] = None
+       finally:
+               f.close()
+       return allmodules.keys()
+
+
+# Find the file containing a module, given its name.
+# Return filename, or '<builtin>', or '<unknown>'.
+
+builtins = sys.builtin_module_names
+if 'sys' not in builtins: builtins.append('sys')
+# XXX this table may have to be changed depending on your platform:
+tails = ['.so', 'module.so', '.py', '.pyc']
+
+def findmodule(modname, path = sys.path):
+       if modname in builtins: return '<builtin>'
+       for dirname in path:
+               for tail in tails:
+                       fullname = os.path.join(dirname, modname + tail)
+                       try:
+                               f = open(fullname, 'r')
+                       except IOError:
+                               continue
+                       f.close()
+                       return fullname
+       return '<unknown>'
+
+
+# Test the above functions.
+
+def test():
+       if not sys.argv[1:]:
+               print 'usage: python findmodules.py scriptfile [morefiles ...]'
+               sys.exit(2)
+       done = findmodules(sys.argv[1], sys.argv[2:])
+       items = done.items()
+       items.sort()
+       for mod, file in [('Module', 'File')] + items:
+               print "%-15s %s" % (mod, file)
+
+if __name__ == '__main__':
+       test()

diff --git a/Tools/freeze/freeze.py b/Tools/freeze/freeze.py
index 52285a6d65bc3438a23700856df4dc33447123cb..2588c6023244bd3b04b0d1dedde22c40fe525854 100755 (executable)
--- a/Tools/freeze/freeze.py
+++ b/Tools/freeze/freeze.py
@@ -1,576 +1,176 @@
 #! /usr/local/bin/python
 
-# Given a Python script, create a binary that runs the script.
-# The binary is 100% independent of Python libraries and binaries.
-# It will not contain any Python source code -- only "compiled" Python
-# (as initialized static variables containing marshalled code objects).
-# It even does the right thing for dynamically loaded modules!
-# The module search path of the binary is set to the current directory.
-#
-# Some problems remain:
-# - You need to have the Python source tree lying around as well as
-#   the various libraries used to generate the Python binary.
-# - For scripts that use many modules it generates absurdly large
-#   files (frozen.c and config.o as well as the final binary),
-#   and is consequently rather slow.
-#
-# Caveats:
-# - The search for modules sometimes finds modules that are never
-#   actually imported since the code importing them is never executed.
-# - If an imported module isn't found, you get a warning but the
-#   process of freezing continues.  The binary will fail if it
-#   actually tries to import one of these modules.
-# - This often happens with the module 'mac', which module 'os' tries
-#   to import (to determine whether it is running on a Macintosh).
-#   You can ignore the warning about this.
-# - If the program dynamically reads or generates Python code and
-#   executes it, this code may reference built-in or library modules
-#   that aren't present in the frozen binary, and this will fail.
-# - Your program may be using external data files, e.g. compiled
-#   forms definitions (*.fd).  These aren't incorporated. By default,
-#   the sys.path in the resulting binary is only '.' (but you can override
-#   that with the -P option).
-#
-# Usage hints:
-# - If you have a bunch of scripts that you want to freeze, instead
-#   of freezing each of them separately, you might consider writing
-#   a tiny main script that looks at sys.argv[0] and then imports
-#   the corresponding module.  You can then make links to the
-#   frozen binary named after the various scripts you support.
-#   Pass the additional scripts as arguments after the main script.
-#   A minimal script to do this is the following.
-#       import sys, posixpath
-#       exec('import ' + posixpath.basename(sys.argv[0]) + '\n')
-#
-# Mods by Jack, August 94:
-# - Removed all static configuration stuff. Now, Setup and Makefile files
-#   are parsed to obtain the linking info for the libraries. You have to
-#   supply the -B option, though.
-# - Added -P (set sys.path) and -I/-D/-L/-l options (passed on to cc and
-#   ld).
+# "Freeze" a Python script into a binary.
+# Usage: see first function below (before the imports!)
+
+# HINTS:
+# - Edit the line at XXX below before running!
+# - You must have done "make inclinstall libainstall" in the Python
+#   build directory.
+# - The script should not use dynamically loaded modules
+#   (*.so on most systems).
+
+
+# XXX Change the following line to point to your Demo/freeze directory!
+pack = '/ufs/guido/src/python/Demo/freeze'
+
+
+# Print usage message and exit
+
+def usage(msg = None):
+       if msg:
+               sys.stderr.write(str(msg) + '\n')
+       sys.stderr.write('usage: freeze [-p prefix] script [module] ...\n')
+       sys.exit(2)
+
+
+# Import standard modules
 
-import os
-import sys
-import regex
 import getopt
-import regsub
+import os
 import string
-import marshal
-
-# Exception used when scanfile fails
-NoSuchFile = 'NoSuchFile'
-
-# Global options
-builddir = ''                          # -B dir
-quiet = 0                              # -q
-verbose = 0                            # -v
-noexec = 0                             # -n
-nowrite = 0                            # -N
-ofile = 'a.out'                                # -o file
-path = '\'"."\''                       # -P path
-
-cc_options = []                                # Collects cc options
-ld_options = []                                # Collects ld options
-module_libraries = {}                  # ld options for each module
-global_libraries = []                  # Libraries we always need
-include_path = ''                      # Include path, from Makefile 
-lib_path = ''                          # and lib path, ditto
-compiler = 'cc'                                # and compiler
-
-# Main program -- argument parsing etc.
+import sys
+import addpack
+
+
+# Set the directory to look for the freeze-private modules
+
+dir = os.path.dirname(sys.argv[0])
+if dir:
+       pack = dir
+addpack.addpack(pack)
+
+
+# Import the freeze-private modules
+
+import findmodules
+import makeconfig
+import makefreeze
+import makemakefile
+import parsesetup
+
+hint = """
+Use the '-p prefix' command line option to specify the prefix used
+when you ran 'Make inclinstall libainstall' in the Python build directory.
+(Please specify an absolute path.)
+"""
+
+
+# Main program
+
 def main():
-       global quiet, verbose, noexec, nowrite, ofile, builddir, path
+       # overridable context
+       prefix = '/usr/local'           # settable with -p option
+       path = sys.path
+
+       # output files
+       frozen_c = 'frozen.c'
+       config_c = 'config.c'
+       target = 'a.out'                # normally derived from script name
+       makefile = 'Makefile'
+
+       # parse command line
        try:
-               opts, args = getopt.getopt(sys.argv[1:], 'B:nNo:P:qvI:D:L:l:')
+               opts, args = getopt.getopt(sys.argv[1:], 'p:')
+               if not args:
+                       raise getopt.error, 'not enough arguments'
        except getopt.error, msg:
-               usage(str(msg))
-               sys.exit(2)
+               usage('getopt error: ' + str(msg))
+
+       # proces option arguments
        for o, a in opts:
-               if o == '-B': builddir = a
-               if o == '-n': noexec = 1
-               if o == '-N': nowrite = 1
-               if o == '-o': ofile = a
-               if o == '-P':
-                   if '"' in a:
-                       usage('sorry, cannot have " in -P option')
-                       sys.exit(2)
-                   path = `'"' + a + '"'`
-               if o == '-q': verbose = 0; quiet = 1
-               if o == '-v': verbose = verbose + 1; quiet = 0
-               if o in ('-I', '-D'): cc_options.append(o+a)
-               if o in ('-L', '-l'): ld_options.append(o+a)
-       if not builddir:
-           usage('sorry, you have to pass a -B option')
-           sys.exit(2)
-       if len(args) < 1:
-               usage('please pass at least one file argument')
-               sys.exit(2)
-       process(args[0], args[1:])
-
-# Print usage message to stderr
-def usage(*msgs):
-       sys.stdout = sys.stderr
-       for msg in msgs: print msg
-       print 'Usage: freeze [options] scriptfile [modulefile ...]'
-       print '-B dir  : name of python build dir (no default)'
-       print '-n      : generate the files but don\'t compile and link'
-       print '-N      : don\'t write frozen.c (do compile unless -n given)'
-       print '-o file : binary output file (default a.out)'
-       print '-P path : set sys.path for program (default ".")'
-       print '-q      : quiet (no messages at all except errors)'
-       print '-v      : verbose (lots of extra messages)'
-       print '-D and -I options are passed to cc, -L and -l to ld'
-
-# Process the script file
-def process(filename, addmodules):
-       global noexec
-       #
-       if not quiet: print 'Computing needed modules ...'
-       todo = {}
-       todo['__main__'] = filename
-       for name in addmodules:
-               mod = os.path.basename(name)
-               if mod[-3:] == '.py': mod = mod[:-3]
-               todo[mod] = name
-       try:
-               dict = closure(todo)
-       except NoSuchFile, filename:
-               sys.stderr.write('Can\'t open file %s\n' % filename)
-               sys.exit(1)
-       #
-       mods = dict.keys()
-       mods.sort()
-       #
-       if verbose:
-               print '%-15s %s' % ('Module', 'Filename')
-               for mod in mods:
-                       print '%-15s %s' % (`mod`, dict[mod])
-       #
-       if not quiet: print 'Looking for dynamically linked modules ...'
-       dlmodules = []
-       objs = []
-       libs = []
-       for mod in mods:
-               if dict[mod][-2:] == '.o':
-                       if verbose: print 'Found', mod, dict[mod]
-                       dlmodules.append(mod)
-                       objs.append(dict[mod])
-                       libsname = dict[mod][:-2] + '.libs'
-                       try:
-                               f = open(libsname, 'r')
-                       except IOError:
-                               f = None
-                       if f:
-                               libtext = f.read()
-                               f.close()
-                               for lib in string.split(libtext):
-                                       if lib in libs: libs.remove(lib)
-                                       libs.append(lib)
-       #
-       if not nowrite:
-               if not quiet: print 'Writing frozen.c ...'
-               writefrozen('frozen.c', dict)
-       else:
-               if not quiet: print 'NOT writing frozen.c ...'
-       #
-       if not quiet:
-           print 'Deducing compile/link options from', builddir
-       #
-       # Parse the config info
-       #
-       parse(builddir)
-       CONFIG_IN = lib_path + '/config.c.in'
-       FMAIN = lib_path + '/frozenmain.c'
-       CC = compiler
-       #
-##     if not dlmodules:
-       if 0:
-               config = CONFIG
-               if not quiet: print 'Using existing', config, '...'
-       else:
-               config = 'tmpconfig.c'
-               if nowrite:
-                       if not quiet: print 'NOT writing config.c ...'
+               if o == '-p':
+                       prefix = a
+
+       # locations derived from options
+       binlib = os.path.join(prefix, 'lib/python/lib')
+       incldir = os.path.join(prefix, 'include/Py')
+       config_c_in = os.path.join(binlib, 'config.c.in')
+       frozenmain_c = os.path.join(binlib, 'frozenmain.c')
+       makefile_in = os.path.join(binlib, 'Makefile')
+       defines = ['-DHAVE_CONFIG_H', '-DUSE_FROZEN', '-DNO_MAIN',
+                  '-DPTHONPATH=\\"$(PYTHONPATH)\\"']
+       includes = ['-I' + incldir, '-I' + binlib]
+
+       # sanity check of locations
+       for dir in prefix, binlib, incldir:
+               if not os.path.exists(dir):
+                       usage('needed directory %s not found' % dir + hint)
+               if not os.path.isdir(dir):
+                       usage('%s: not a directory' % dir)
+       for file in config_c_in, makefile_in, frozenmain_c:
+               if not os.path.exists(file):
+                       usage('needed file %s not found' % file)
+               if not os.path.isfile(file):
+                       usage('%s: not a plain file' % file)
+
+       # check that file arguments exist
+       for arg in args:
+               if not os.path.exists(arg):
+                       usage('argument %s not found' % arg)
+               if not os.path.isfile(arg):
+                       usage('%s: not a plain file' % arg)
+
+       # process non-option arguments
+       scriptfile = args[0]
+       modules = args[1:]
+
+       # derive target name from script name
+       base = os.path.basename(scriptfile)
+       base, ext = os.path.splitext(base)
+       if base:
+               if base != scriptfile:
+                       target = base
                else:
-                       if not quiet:
-                               print 'Writing config.c with dl modules ...'
-                       f = open(CONFIG_IN, 'r')
-                       g = open(config, 'w')
-                       m1 = regex.compile('-- ADDMODULE MARKER 1 --')
-                       m2 = regex.compile('-- ADDMODULE MARKER 2 --')
-                       builtinmodules = []
-                       stdmodules = ('sys', '__main__', '__builtin__',
-                                     'marshal')
-                       for mod in dict.keys():
-                               if dict[mod] == '<builtin>' and \
-                                         mod not in stdmodules:
-                                       builtinmodules.append(mod)
-                       todomodules = builtinmodules + dlmodules
-                       while 1:
-                               line = f.readline()
-                               if not line: break
-                               g.write(line)
-                               if m1.search(line) >= 0:
-                                       if verbose: print 'Marker 1 ...'
-                                       for mod in todomodules:
-                                               g.write('extern void init' + \
-                                                 mod + '();\n')
-                               if m2.search(line) >= 0:
-                                       if verbose: print 'Marker 2 ...'
-                                       for mod in todomodules:
-                                               g.write('{"' + mod + \
-                                                 '", init' + mod + '},\n')
-                       g.close()
-       #
-       if not quiet:
-               if noexec: print 'Generating compilation commands ...'
-               else: print 'Starting compilation ...'
-       defs = ['-DNO_MAIN', '-DUSE_FROZEN']
-       defs.append('-DPYTHONPATH='+path)
-       #
-       incs = ['-I.', '-I' + include_path]
-#      if dict.has_key('stdwin'):
-#              incs.append('-I' + j(STDWIN, 'H'))
-       #
-       srcs = [config, FMAIN]
-       #
-       modlibs = module_libraries
-       
+                       target = base + '.bin'
+
+       # Actual work starts here...
+
+       dict = findmodules.findmodules(scriptfile, modules, path)
+
+       builtins = []
        for mod in dict.keys():
-           if modlibs.has_key(mod):
-               libs = libs + modlibs[mod]
-
-       libs = libs + global_libraries
-       #
-       # remove dups:
-       # XXXX Not well tested...
-       nskip = 0
-       newlibs = []
-       while libs:
-           l = libs[0]
-           del libs[0]
-           if l[:2] == '-L' and l in newlibs:
-               nskip = nskip + 1
-               continue
-           if (l[:2] == '-l' or l[-2:] == '.a') and l in libs:
-               nskip = nskip + 1
-               continue
-           newlibs.append(l)
-       libs = newlibs
-       if nskip and not quiet:
-           print 'Removed %d duplicate libraries'%nskip
-       #
-       sts = 0
-       #
-       cmd = CC + ' -c'
-       if cc_options:
-           cmd = cmd + ' ' + string.join(cc_options)
-       cmd = cmd + ' ' + string.join(defs)
-       cmd = cmd + ' ' + string.join(incs)
-       cmd = cmd + ' ' + string.join(srcs)
-       print cmd
-       #
-       if not noexec:
-               sts = os.system(cmd)
-               if sts:
-                       print 'Exit status', sts, '-- turning on -n'
-                       noexec = 1
-       #
-       for s in srcs:
-               s = os.path.basename(s)
-               if s[-2:] == '.c': s = s[:-2]
-               o = s + '.o'
-               objs.insert(0, o)
-       #
-       cmd = CC
-       cmd = cmd + ' ' + string.join(objs)
-       cmd = cmd + ' ' + string.join(libs)
-       if ld_options:
-           cmd = cmd + ' ' + string.join(ld_options)
-       cmd = cmd + ' -o ' + ofile
-       print cmd
-       #
-       if not noexec:
-               sts = os.system(cmd)
-               if sts:
-                       print 'Exit status', sts
-               else:
-                       print 'Done.'
-       #
-       if not quiet and not noexec and sts == 0:
-               print 'Note: consider this:'; print '\tstrip', ofile
-       #
-       sys.exit(sts)
-
-
-# Generate code for a given module
-def makecode(filename):
-       if filename[-2:] == '.o':
-               return None
+               if dict[mod] == '<builtin>':
+                       builtins.append(mod)
+
+       outfp = open(frozen_c, 'w')
        try:
-               f = open(filename, 'r')
-       except IOError:
-               return None
-       if verbose: print 'Making code from', filename, '...'
-       text = f.read()
-       code = compile(text, filename, 'exec')
-       f.close()
-       return marshal.dumps(code)
-
-
-# Write the C source file containing the frozen Python code
-def writefrozen(filename, dict):
-       f = open(filename, 'w')
-       codelist = []
-       for mod in dict.keys():
-               codestring = makecode(dict[mod])
-               if codestring is not None:
-                       codelist.append((mod, codestring))
-       write = sys.stdout.write
-       save_stdout = sys.stdout
+               makefreeze.makefreeze(outfp, dict)
+       finally:
+               outfp.close()
+
+       infp = open(config_c_in)
+       outfp = open(config_c, 'w')
        try:
-               sys.stdout = f
-               for mod, codestring in codelist:
-                       if verbose:
-                               write('Writing initializer for %s\n'%mod)
-                       print 'static unsigned char M_' + mod + '[' + \
-                                 str(len(codestring)) + '+1] = {'
-                       for i in range(0, len(codestring), 16):
-                               for c in codestring[i:i+16]:
-                                       print str(ord(c)) + ',',
-                               print
-                       print '};'
-               print 'struct frozen {'
-               print '  char *name;'
-               print '  unsigned char *code;'
-               print '  int size;'
-               print '} frozen_modules[] = {'
-               for mod, codestring in codelist:
-                       print '  {"' + mod + '",',
-                       print 'M_' + mod + ',',
-                       print str(len(codestring)) + '},'
-               print '  {0, 0, 0} /* sentinel */'
-               print '};'
+               makeconfig.makeconfig(infp, outfp, builtins)
        finally:
-               sys.stdout = save_stdout
-       f.close()
-
-
-# Determine the names and filenames of the modules imported by the
-# script, recursively.  This is done by scanning for lines containing
-# import statements.  (The scanning has only superficial knowledge of
-# Python syntax and no knowledge of semantics, so in theory the result
-# may be incorrect -- however this is quite unlikely if you don't
-# intentionally obscure your Python code.)
-
-# Compute the closure of scanfile() -- special first file because of script
-def closure(todo):
-       done = {}
-       while todo:
-               newtodo = {}
-               for modname in todo.keys():
-                       if not done.has_key(modname):
-                               filename = todo[modname]
-                               if filename is None:
-                                       filename = findmodule(modname)
-                               done[modname] = filename
-                               if filename in ('<builtin>', '<unknown>'):
-                                       continue
-                               modules = scanfile(filename)
-                               for m in modules:
-                                       if not done.has_key(m):
-                                               newtodo[m] = None
-               todo = newtodo
-       return done
-
-# Scan a file looking for import statements
-importstr = '\(^\|:\)[ \t]*import[ \t]+\([a-zA-Z0-9_, \t]+\)'
-fromstr   = '\(^\|:\)[ \t]*from[ \t]+\([a-zA-Z0-9_]+\)[ \t]+import[ \t]+'
-isimport = regex.compile(importstr)
-isfrom = regex.compile(fromstr)
-def scanfile(filename):
-       allmodules = {}
+               outfp.close()
+       infp.close()
+
+       cflags = defines + includes + ['$(OPT)']
+       libs = []
+       for n in 'Modules', 'Python', 'Objects', 'Parser':
+               n = 'lib%s.a' % n
+               n = os.path.join(binlib, n)
+               libs.append(n)
+
+       makevars = parsesetup.getmakevars(makefile_in)
+       somevars = {}
+       for key in makevars.keys():
+               somevars[key] = makevars[key]
+
+       somevars['CFLAGS'] = string.join(cflags) # override
+       files = ['$(OPT)', config_c, frozenmain_c] + libs + \
+               ['$(MODLIBS)', '$(LIBS)', '$(SYSLIBS)']
+
+       outfp = open(makefile, 'w')
        try:
-               f = open(filename, 'r')
-       except IOError, msg:
-               raise NoSuchFile, filename
-       while 1:
-               line = f.readline()
-               if not line: break # EOF
-               while line[-2:] == '\\\n': # Continuation line
-                       line = line[:-2] + ' '
-                       line = line + f.readline()
-               if isimport.search(line) >= 0:
-                       rawmodules = isimport.group(2)
-                       modules = string.splitfields(rawmodules, ',')
-                       for i in range(len(modules)):
-                               modules[i] = string.strip(modules[i])
-               elif isfrom.search(line) >= 0:
-                       modules = [isfrom.group(2)]
-               else:
-                       continue
-               for mod in modules:
-                       allmodules[mod] = None
-       f.close()
-       return allmodules.keys()
-
-# Find the file containing a module, given its name; None if not found
-builtins = sys.builtin_module_names + ['sys']
-def findmodule(modname):
-       if modname in builtins: return '<builtin>'
-       for dirname in sys.path:
-               dlfullname = os.path.join(dirname, modname + 'module.o')
-               try:
-                       f = open(dlfullname, 'r')
-               except IOError:
-                       f = None
-               if f:
-                       f.close()
-                       return dlfullname
-               fullname = os.path.join(dirname, modname + '.py')
-               try:
-                       f = open(fullname, 'r')
-               except IOError:
-                       continue
-               f.close()
-               return fullname
-       if not quiet:
-               sys.stderr.write('Warning: module %s not found\n' % modname)
-       return '<unknown>'
-#
-# Parse a setup file. Returns two dictionaries, one containing variables
-# defined with their values and one containing module definitions
-#
-def parse_setup(fp):
-    modules = {}
-    variables = {}
-    for line in fp.readlines():
-       if '#' in line:                         # Strip comments
-           line = string.splitfields(line, '#')[0]
-       line = string.strip(line[:-1])          # Strip whitespace
-       if not line:
-           continue
-       words = string.split(line)
-       if '=' in words[0]:
-           #
-           # equal sign before first space. Definition
-           #
-           pos = string.index(line, '=')
-           name = line[:pos]
-           value = string.strip(line[pos+1:])
-           variables[name] = value
-       else:
-           modules[words[0]] = words[1:]
-    return modules, variables
-#
-# Parse a makefile. Returns a list of the variables defined.
-#
-def parse_makefile(fp):
-    variables = {}
-    for line in fp.readlines():
-       if '#' in line:                         # Strip comments
-           line = string.splitfields(line, '#')[0]
-       if not line:
-           continue
-       if line[0] in string.whitespace:
-           continue
-       line = string.strip(line[:-1])          # Strip whitespace
-       if not line:
-           continue
-       if '=' in string.splitfields(line, ':')[0]: 
-           #
-           # equal sign before first colon. Definition
-           #
-           pos = string.index(line, '=')
-           name = line[:pos]
-           value = string.strip(line[pos+1:])
-           variables[name] = value
-    return variables
-
-#
-# Recursively add loader options from Setup files in extension
-# directories.
-#
-def add_extension_directory(name, isinstalldir):
-    if verbose:
-       print 'Adding extension directory', name
-    fp = open(name + '/Setup', 'r')
-    modules, variables = parse_setup(fp)
-    #
-    # Locate all new modules and remember the ld flags needed for them
-    #
-    for m in modules.keys():
-       if module_libraries.has_key(m):
-           continue
-       options = modules[m]
-       if isinstalldir:
-           ld_options = []
-       else:
-           ld_options = [name + '/lib.a']
-       for o in options:
-           # ld options are all capital except DUIC and l
-           if o[:-2] == '.a':
-               ld_options.append(o)
-           elif o[0] == '-':
-               if o[1] == 'l':
-                   ld_options.append(o)
-               elif o[1] in string.uppercase and not o[1] in 'DUIC':
-                   ld_options.append(o)
-       module_libraries[m] = ld_options
-    #
-    # See if we have to bother with base setups
-    #
-    if variables.has_key('BASESETUP'):
-       if isinstalldir:
-           raise 'installdir has base setup'
-       setupfiles = string.split(variables['BASESETUP'])
-       for s in setupfiles:
-           if s[-6:] <> '/Setup':
-               raise 'Incorrect BASESETUP', s
-           s = s[:-6]
-           if s[0] <> '/':
-               s = name + '/' + s
-               s = os.path.normpath(s)
-           add_extension_directory(s, 0)
-#
-# Main routine for this module: given a build directory, get all
-# information needed for the linker.
-#
-def parse(dir):
-    global include_path
-    global lib_path
-    global compiler
-    
-    fp = open(dir + '/Makefile', 'r')
-    #
-    # First find the global libraries and the base python
-    #
-    vars = parse_makefile(fp)
-    if vars.has_key('CC'):
-       compiler = vars['CC']
-    if not vars.has_key('installdir'):
-       raise 'No $installdir in Makefile'
-    include_path = vars['installdir'] + '/include/Py'
-    lib_path = vars['installdir'] + '/lib/python/lib'
-    global_libraries.append('-L' + lib_path)
-    global_libraries.append('-lPython')
-    global_libraries.append('-lParser')
-    global_libraries.append('-lObjects')
-    global_libraries.append('-lModules')
-    for name in ('LIBS', 'LIBM', 'LIBC'):
-       if not vars.has_key(name):
-           raise 'Missing required def in Makefile', name
-       for lib in string.split(vars[name]):
-           global_libraries.append(lib)
-    #
-    # Next, parse the modules from the base python
-    #
-    add_extension_directory(lib_path, 1)
-    #
-    # Finally, parse the modules from the extension python
-    #
-    if dir <> lib_path:
-       add_extension_directory(dir, 0)
-
-# Call the main program
+               makemakefile.makemakefile(outfp, somevars, files, target)
+       finally:
+               outfp.close()
+
+       # Done!
+
+       print 'Now run make to build the target:', target
+
 main()

diff --git a/Tools/freeze/makeconfig.py b/Tools/freeze/makeconfig.py
new file mode 100644 (file)
index 0000000..958c2be
--- /dev/null
+++ b/Tools/freeze/makeconfig.py
@@ -0,0 +1,57 @@
+import regex
+
+
+# Write the config.c file
+
+never = ['marshal', '__main__', '__builtin__', 'sys']
+
+def makeconfig(infp, outfp, modules):
+       m1 = regex.compile('-- ADDMODULE MARKER 1 --')
+       m2 = regex.compile('-- ADDMODULE MARKER 2 --')
+       while 1:
+               line = infp.readline()
+               if not line: break
+               outfp.write(line)
+               if m1 and m1.search(line) >= 0:
+                       m1 = None
+                       for mod in modules:
+                               if mod in never:
+                                       continue
+                               outfp.write('extern void init%s();\n' % mod)
+               elif m2 and m2.search(line) >= 0:
+                       m2 = None
+                       for mod in modules:
+                               if mod in never:
+                                       continue
+                               outfp.write('\t{"%s", init%s},\n' %
+                                           (mod, mod))
+       if m1:
+               sys.stderr.write('MARKER 1 never found\n')
+       elif m2:
+               sys.stderr.write('MARKER 2 never found\n')
+
+
+# Test program.
+
+def test():
+       import sys
+       if not sys.argv[3:]:
+               print 'usage: python makeconfig.py config.c.in outputfile',
+               print 'modulename ...'
+               sys.exit(2)
+       if sys.argv[1] == '-':
+               infp = sys.stdin
+       else:
+               infp = open(sys.argv[1])
+       if sys.argv[2] == '-':
+               outfp = sys.stdout
+       else:
+               outfp = open(sys.argv[2], 'w')
+       makeconfig(infp, outfp, sys.argv[3:])
+       if outfp != sys.stdout:
+               outfp.close()
+       if infp != sys.stdin:
+               infp.close()
+
+if __name__ == '__main__':
+       test()

diff --git a/Tools/freeze/makefreeze.py b/Tools/freeze/makefreeze.py
new file mode 100644 (file)
index 0000000..4a3da99
--- /dev/null
+++ b/Tools/freeze/makefreeze.py
@@ -0,0 +1,91 @@
+import marshal
+
+
+# Write a file containing frozen code for the modules in the dictionary.
+
+header = """
+struct frozen {
+       char *name;
+       unsigned char *code;
+       int size;
+} frozen_modules[] = {
+"""
+trailer = """\
+       {0, 0, 0} /* sentinel */
+};
+"""
+
+def makefreeze(outfp, dict):
+       done = []
+       mods = dict.keys()
+       mods.sort()
+       for mod in mods:
+               modfn = dict[mod]
+               try:
+                       str = makecode(modfn)
+               except IOError, msg:
+                       sys.stderr.write("%s: %s\n" % (modfn, str(msg)))
+                       continue
+               if str:
+                       done.append(mod, len(str))
+                       writecode(outfp, mod, str)
+       outfp.write(header)
+       for mod, size in done:
+               outfp.write('\t{"%s", M_%s, %d},\n' % (mod, mod, size))
+       outfp.write(trailer)
+
+
+# Return code string for a given module -- either a .py or a .pyc
+# file.  Return either a string or None (if it's not Python code).
+# May raise IOError.
+
+def makecode(filename):
+       if filename[-3:] == '.py':
+               f = open(filename, 'r')
+               try:
+                       text = f.read()
+                       code = compile(text, filename, 'exec')
+               finally:
+                       f.close()
+               return marshal.dumps(code)
+       if filename[-4:] == '.pyc':
+               f = open(filename, 'rb')
+               try:
+                       f.seek(8)
+                       str = f.read()
+               finally:
+                       f.close()
+               return str
+       # Can't generate code for this extension
+       return None
+
+
+# Write a C initializer for a module containing the frozen python code.
+# The array is called M_<mod>.
+
+def writecode(outfp, mod, str):
+       outfp.write('static unsigned char M_%s[] = {' % mod)
+       for i in range(0, len(str), 16):
+               outfp.write('\n\t')
+               for c in str[i:i+16]:
+                       outfp.write('%d,' % ord(c))
+       outfp.write('\n};\n')
+
+
+# Test for the above functions.
+
+def test():
+       import os
+       import sys
+       if not sys.argv[1:]:
+               print 'usage: python freezepython.py file.py(c) ...'
+               sys.exit(2)
+       dict = {}
+       for arg in sys.argv[1:]:
+               base = os.path.basename(arg)
+               mod, ext = os.path.splitext(base)
+               dict[mod] = arg
+       makefreeze(sys.stdout, dict)
+
+if __name__ == '__main__':
+       test()

diff --git a/Tools/freeze/makemakefile.py b/Tools/freeze/makemakefile.py
new file mode 100644 (file)
index 0000000..5418f93
--- /dev/null
+++ b/Tools/freeze/makemakefile.py
@@ -0,0 +1,27 @@
+# Write the actual Makefile.
+
+import os
+import string
+
+def makemakefile(outfp, makevars, files, target):
+       outfp.write("# Makefile generated by freeze.py script\n\n")
+
+       keys = makevars.keys()
+       keys.sort()
+       for key in keys:
+               outfp.write("%s=%s\n" % (key, makevars[key]))
+       outfp.write("\nall: %s\n" % target)
+
+       deps = []
+       for i in range(len(files)):
+               file = files[i]
+               if file[-2:] == '.c':
+                       base = os.path.basename(file)
+                       dest = base[:-2] + '.o'
+                       outfp.write("%s: %s\n" % (dest, file))
+                       outfp.write("\t$(CC) $(CFLAGS) -c %s\n" % file)
+                       files[i] = dest
+                       deps.append(dest)
+
+       outfp.write("\n%s: %s\n" % (target, string.join(deps)))
+       outfp.write("\t$(CC) %s -o %s\n" % (string.join(files), target))

diff --git a/Tools/freeze/parsesetup.py b/Tools/freeze/parsesetup.py
new file mode 100644 (file)
index 0000000..1795671
--- /dev/null
+++ b/Tools/freeze/parsesetup.py
@@ -0,0 +1,98 @@
+# Parse Makefiles and Python Setup(.in) files.
+
+import regex
+import string
+
+
+# Extract variable definitions from a Makefile.
+# Return a dictionary mapping names to values.
+# May raise IOError.
+
+makevardef = regex.compile('^\([a-zA-Z0-9_]+\)[ \t]*=\(.*\)')
+
+def getmakevars(filename):
+       variables = {}
+       fp = open(filename)
+       try:
+               while 1:
+                       line = fp.readline()
+                       if not line:
+                               break
+                       if makevardef.match(line) < 0:
+                               continue
+                       name, value = makevardef.group(1, 2)
+                       # Strip trailing comment
+                       i = string.find(value, '#')
+                       if i >= 0:
+                               value = value[:i]
+                       value = string.strip(value)
+                       variables[name] = value
+       finally:
+               fp.close()
+       return variables
+
+
+# Parse a Python Setup(.in) file.
+# Return two dictionaries, the first mapping modules to their
+# definitions, the second mapping variable names to their values.
+# May raise IOError.
+
+setupvardef = regex.compile('^\([a-zA-Z0-9_]+\)=\(.*\)')
+
+def getsetupinfo(filename):
+       modules = {}
+       variables = {}
+       fp = open(filename)
+       try:
+               while 1:
+                       line = fp.readline()
+                       if not line:
+                               break
+                       # Strip comments
+                       i = string.find(line, '#')
+                       if i >= 0:
+                               line = line[:i]
+                       if setupvardef.match(line) >= 0:
+                               name, value = setupvardef.group(1, 2)
+                               variables[name] = string.strip(value)
+                       else:
+                               words = string.split(line)
+                               if words:
+                                       modules[words[0]] = words[1:]
+       finally:
+               fp.close()
+       return modules, variables
+
+
+# Test the above functions.
+
+def test():
+       import sys
+       import os
+       if not sys.argv[1:]:
+               print 'usage: python parsesetup.py Makefile*|Setup* ...'
+               sys.exit(2)
+       for arg in sys.argv[1:]:
+               base = os.path.basename(arg)
+               if base[:8] == 'Makefile':
+                       print 'Make style parsing:', arg
+                       v = getmakevars(arg)
+                       prdict(v)
+               elif base[:5] == 'Setup':
+                       print 'Setup style parsing:', arg
+                       m, v = getsetupinfo(arg)
+                       prdict(m)
+                       prdict(v)
+               else:
+                       print arg, 'is neither a Makefile nor a Setup file'
+                       print '(name must begin with "Makefile" or "Setup")'
+
+def prdict(d):
+       keys = d.keys()
+       keys.sort()
+       for key in keys:
+               value = d[key]
+               print "%-15s" % key, str(value)
+
+if __name__ == '__main__':
+       test()