# Part of the A-A-P recipe executive: Parse and execute recipe commands

# Copyright (C) 2002 Stichting NLnet Labs
# Permission to copy and use this file is specified in the file COPYING.
# If this file is missing you can find it here: http://www.a-a-p.org/COPYING

import string
import sys
import traceback

from Util import *
from Work import setrpstack
from Error import *
from RecPos import rpcopy
import Global

# ":" commands that only work at the toplevel
aap_cmd_toplevel = [
		"autodepend",
		"recipe",
		"rule",
		"variant",
		]

# All possible names of ":" commands in a recipe, including toplevel-only ones.
aap_cmd_names = [
		"add",
		"attr",
		"attribute",
		"cat",
		"checkin",
		"checkout",
		"child",
		"commit",
		"commitall",
		"copy",
		"del",
		"delete",
		"error",
		"export",
		"filetype",
		"include",
		"mkdir",
		"move",
		"print",
		"publish",
		"publishall",
		"refresh",
		"remove",
		"removeall",
		"require",
		"sys",
		"system",
		"touch",
		"unlock",
		"update",
		"verscont",
		] + aap_cmd_toplevel

# marker for recipe line number in Python script
line_marker = '#@recipe='
line_marker_len = len(line_marker)


def assert_var_name(name, rpstack):
    """Check if "name" is a valid variable name.
       If it isn't, throw a user exception."""
    for c in name:
	if not varchar(c):
	    recipe_error(rpstack, _("Invalid character in variable name"))


def get_var_name(fp):
    """Get the name of a variable from the current position at "fp" and
    get the index of the next non-white after it.  Returns an empty string if
    there is no valid variable name."""
    idx = fp.idx
    while idx < fp.line_len and varchar(fp.line[idx]):
	idx = idx + 1
    return fp.line[fp.idx:idx], skip_white(fp.line, idx)


class ArgItem:
    """Object used as smallest part in the arglist."""
    def __init__(self, isexpr, str):
	self.isexpr = isexpr	    # 0 for string, 1 for Python expression
	self.str = str		    # the string itself


def getarg(fp, stop, globals):
    """Get an argument starting at fp.line[fp.idx] and ending at a character in
       stop[].
       Quotes are used to include stop characters in the argument.
       Backticks are handled here.  `` is reduced to a single `.  A Python
       expression `python` is translated to '" + expr2str(python) + "'.
       Returns the resulting string and fp.idx is updated to the character
       after the argument (the stop character or past the end of line).
       """
    res = ''				# argument collected so far
    inquote = ''			# quote we're inside of
    inbraces = 0			# inside {} count
    while 1:
	if fp.idx >= fp.line_len:	# end of line
	    break

	# Python `expression`?
	if fp.line[fp.idx] == '`':
	    # `` isn't the start of an expression, reduce it to a single `.
	    if fp.idx + 1 < fp.line_len and fp.line[fp.idx + 1] == '`':
		res = res + '`'
		fp.idx = fp.idx + 2
		continue
	    # Append the Python expression.
	    res = res + '" + expr2str(' + get_py_expr(fp) + ') + "'
	    continue

	# End of quoted string?
	if inquote:
	    if fp.line[fp.idx] == inquote:
		inquote = ''

	# Start of quoted string?
	elif fp.line[fp.idx] == '"' or fp.line[fp.idx] == "'":
	    inquote = fp.line[fp.idx]

	else:
	    # start/end of {}?
	    if fp.line[fp.idx] == '{':
		inbraces = inbraces + 1
	    elif fp.line[fp.idx] == '}':
		inbraces = inbraces - 1
		if inbraces < 0:
		    # TODO: recipe_error(fp.rpstack, _("Unmatched }"))
		    inbraces = 0

	    # Stop character found?
	    # A ':' must be followed by white space to be recongized.
	    # A '=' must not be inside {}.
	    if string.find(stop, fp.line[fp.idx]) != -1 \
		    and (fp.line[fp.idx] != ':'
			    or fp.idx + 1 == fp.line_len
			    or fp.line[fp.idx + 1] == ' '
			    or fp.line[fp.idx + 1] == '\t') \
		    and (fp.line[fp.idx] != '=' or inbraces == 0):
		break

	# Need to escape backslash and double quote.
	c = fp.line[fp.idx]
	if c == '"' or c == '\\':
	    res = res + '\\'
	res = res + c
	fp.idx = fp.idx + 1

	# Skip over $$ and $#.
	if (c == '$' and fp.idx < fp.line_len
		       and (fp.line[fp.idx] == '$' or fp.line[fp.idx] == '#')):
	    res = res + fp.line[fp.idx]
	    fp.idx = fp.idx + 1

    # Remove trailing white space.
    e = len(res)
    while e > 0 and is_white(res[e - 1]):
	e = e - 1

    return res[:e]


def get_func_args(fp, indent, globals):
    """Get the arguments for an aap_ function or assignment from the recipe
       line(s).
       Stop at a line with an indent of "indent" or less.

       Input lines:   cmdname arg ` <python-expr> `arg
			        arg
       Result:        "arg " + expr2str(<python-expr>) + "arg arg"

       Return the argument string, advance fp to the following line."""

    res = ''
    fp.idx = skip_white(fp.line, fp.idx)

    while 1:
	if fp.idx >= fp.line_len or fp.line[fp.idx] == '#':
	    # Read the next line
	    fp.nextline()
	    if fp.line is None:
		break	# end of file
	    fp.idx = skip_white(fp.line, 0)
	    if get_indent(fp.line) > indent:
		continue
	    # A line with less indent finishes the list of arguments
	    break

	# Get the argument, stop at a comment, handle python expression.
	# A line break is changed into a space.
	if res:
	    res = res + ' '
	res = res + getarg(fp, "#", globals)

    return '"' + res + '"'


def esc_quote(s):
    """Escape double quotes and backslash with a backslash."""
    return string.replace(string.replace(s, '\\', '\\\\'), '"', '\\"')


def get_commands(fp, indent):
    """Read command lines for a dependency or a rule.
       Stop when the indent is at or below "indent".
       Returns the string of commands, each line ending in '\n'."""
    s = ''
    while 1:
	if fp.line is None or get_indent(fp.line) <= indent:
	    break	    # end of commands reached
	s = s + fp.line + '\n'
	fp.nextline()

    return '"""' + esc_quote(s) + '"""'


def get_py_expr(fp):
    """Get a Python expression from ` to matching `.
       Reduce `` to `.
       fp.idx points to the ` and is advanced to after the matching `.
       Returns the expression excluding the ` before and after."""
    # Remember the RecPos where the first ` was found; need to make a copy,
    # because fp.nextline() will change it.
    rpstack = rpcopy(fp.rpstack, fp.rpstack[-1].line_nr)

    res = ''
    fp.idx = fp.idx + 1
    while 1:
	if fp.idx >= fp.line_len:
	    # Python expression continues in the next line.
	    fp.nextline()
	    if fp.line is None:
		recipe_error(rpstack, _("Missing `"))
	    res = res + '\n'
	if fp.line[fp.idx] == '`':
	    # Either the matching ` or `` that stands for a single `.
	    fp.idx = fp.idx + 1
	    if fp.idx >= fp.line_len or fp.line[fp.idx] != '`':
		break	    # found matching `
	    res = res + '`'
	else:
	    # Append a character to the Python expression.
	    res = res + fp.line[fp.idx]
	fp.idx = fp.idx + 1

    return res


def recipe_error(rpstack, msg):
    """Throw an exception for an error in a recipe:
	    Error: Unknown command
	    in recipe "main.aap" line 88: :foobar asdf
	    included from "main.aap" line 33
       When "rpstack" is empty it's not mentioned, useful for errors
       not related to a specific line."""
    # Note: These messages is not translated, so that a parser for the
    # messages isn't confused by various languages.
    if len(rpstack) == 0:
	e = 'Error in recipe: %s\n' % msg
    else:
	e = 'Error in recipe "%s" line %d: %s\n' \
				 % (rpstack[-1].name, rpstack[-1].line_nr, msg)
    if len(rpstack) > 1:
	for i in range(len(rpstack) - 2, 0, -1):
	    e = e + 'included from "%s" line %d\n' \
					% (rpstack[i].name, rpstack[i].line_nr)
    e = e[:-1]	    # remove trailing \n

    raise UserError, e


def script_error(rpstack, script, e):
    """Handle an error raised while executing the Python script produced for a
    recipe.  The error is probably in the recipe, try to give a useful error
    message."""
    etype, evalue, tb = sys.exc_info()

    # A SyntaxError is special: it's not the last frame in the traceback but
    # only in the "etype" and "evalue".  When there is a filename it must have
    # been an internal error, otherwise it's an error in the converted recipe.
    py_line_nr = -1
    if etype is SyntaxError:
	try:
	    msg, (filename, py_line_nr, offset, line) = evalue
	    if not filename is None:
		py_line_nr = -2
	except:
	    pass

    if py_line_nr < 0:
	# Find the line number in the last traceback frame.
	while tb.tb_next:
	    tb = tb.tb_next
	fname = tb.tb_frame.f_code.co_filename
	if py_line_nr == -2 or (fname and not fname == "<string>"):
	    # If there is a filename, it's not an error in the script.
	    from Main import error_msg
	    error_msg(_("Internal Error"))
	    traceback.print_exc()
	    sys.exit(1)

	py_line_nr = traceback.tb_lineno(tb)

    # Translate the line number in the Python script to the line number
    # in the recipe.
    i = 0
    script_len = len(script)
    rec_line_nr = 1
    while 1:
	if py_line_nr == 1:
	    break
	while i < script_len:
	    if script[i] == '\n':
		break
	    i = i + 1
	i = i + 1
	if i >= script_len:
	    break
	if script[i : i + line_marker_len] == line_marker:
	    i = i + line_marker_len
	    j = i
	    while script[j] in string.digits:
		j = j + 1
	    rec_line_nr = string.atoi(script[i:j])
	py_line_nr = py_line_nr - 1

    # Give the exception error with the line number in the recipe.
    recipe_py_error(rpcopy(rpstack, rec_line_nr), '')


def recipe_py_error(rpstack, msg):
    """Turn the list from format_exception_only() into a simple string and pass
    it to recipe_error()."""
    etype, evalue, tb = sys.exc_info()

    lines = traceback.format_exception_only(etype, evalue)

    # For a syntax error remove the "<string>" and line number that the 
    # Python script causes.
    if etype is SyntaxError:
	try:
	    emsg, (filename, lineno, offset, line) = evalue
	    if filename is None:
		lines[0] = '\n'
	except:
	    pass

    str = msg
    for line in lines[:-1]:
	str = str + line + ' '
    str = str + lines[-1]
    recipe_error(rpstack, str)


def Process(fp, globals):
    """Read all the lines in ParsePos "fp", convert it into a Python script and
       execute it.
       When "fp.string" is empty, the source is a recipe file, otherwise it is
       a string (commands from a dependency or rule)."""

    # Need to be able to find the RecPos stack in globals.
    setrpstack(globals, fp.rpstack)

    class Variant:
	"""Class used to remember nested ":variant" commands in
	   variant_stack."""
	def __init__(self, name, indent):
	    self.name = name
	    self.min_indent = indent
	    self.val_indent = 0
	    self.had_star = 0	    # encountered * item

    #
    # At the start of the loop "fp.line" contains the next line to be
    # processsed.  "fp.rpstack[-1].line_nr" is the number of this line in the
    # recipe.
    #
    script = ""
    shell_cmd = ""	    # shell command collected so far
    variant_stack = []	    # nested ":variant" commands
    had_recipe_cmd = 0	    # encountered ":recipe" command
    fp.nextline()	    # read the first line

    while 1:

	# Skip leading white space (unless at end of file).
	if not fp.line is None:
	    indent = get_indent(fp.line)
	    fp.idx = skip_white(fp.line, 0)

	# If it's not a shell command and the previous line was, generate the
	# collected shell commands now.
	if shell_cmd:
	    if (fp.line is None \
		    or indent < shell_cmd_indent \
		    or fp.line[fp.idx:fp.idx + 4] != ":sys"):
		script = script + (' ' * shell_cmd_indent) \
			      + ('aap_shell(%d, globals(), "%s")\n'
				   % (shell_cmd_line_nr, shell_cmd))
		shell_cmd = ''
	elif not fp.line is None:
	    # Append the recipe line number, used for error messages.
	    script = script + ("%s%d\n" % (line_marker, fp.rpstack[-1].line_nr))


	#
	# Handle the end of commands in a variant or the end of a variant
	#
	if len(variant_stack) > 0:
	    v = variant_stack[-1]
	    if fp.line is None or indent <= v.min_indent:
		# End of the :variant command.
		if v.val_indent == 0:
		    recipe_error(fp.rpstack,
				  _("Exepected list of values after :variant"))
		script = script + (' ' * v.min_indent) + (
					   "BDIR = BDIR + '-' + %s\n" % v.name)
		del variant_stack[-1]
		if len(variant_stack) > 0:
		    continue    # another may end here as well
	    else:
		if v.val_indent == 0:
		    v.val_indent = indent
		    first = 1
		else:
		    first = 0
		if indent <= v.val_indent:
		    # Start of a variant value: "debug [ condition ]"
		    # We simply ignore the condition here.
		    if v.had_star:
			recipe_error(fp.rpstack,
					  _("Variant item * must be last one"))
		    if fp.idx < fp.line_len and fp.line[fp.idx] == '*':
			if (fp.idx + 1 < fp.line_len
					and not is_white(fp.line[fp.idx + 1])):
			    recipe_error(fp.rpstack, _("* must be by itself"))
			if not first:
			    script = script + (' ' * v.min_indent) + "else:\n"
			v.had_star = 1
		    else:
			val, n = get_var_name(fp)
			if val == '':
			    recipe_error(fp.rpstack,
						  _("Exepected variant value"))
			if first:
			    # Specify the default value
			    script = script + (' ' * v.min_indent) + (
				  'if not globals().has_key("%s"):\n' % v.name)
			    script = script + (' ' * v.min_indent) + (
					       '  %s = "%s"\n' % (v.name, val))

			script = script + (' ' * v.min_indent) + (
					    "if %s == '%s':\n" % (v.name, val))
		    fp.nextline()
		    if fp.line is None or get_indent(fp.line) <= v.val_indent:
			script = script + (' ' * v.min_indent) + "pass\n"
		    continue

	#
	# Stop at the end of the file.
	#
	if fp.line is None:
	    break

	#
	# A Python block
	#
	#  recipe:    :python <<<
	#		    command
	#		    command
	#		  <<<
	#  Python:	  if 1:
	#		     command
	#		     command
	#
	if fp.line[fp.idx:fp.idx + 7] == ":python":
	    fp.idx = skip_white(fp.line, fp.idx + 7)
	    if fp.idx >= fp.line_len or fp.line[fp.idx] == '#':
		term = None
	    else:
		n = skip_to_white(fp.line, fp.idx)
		term = fp.line[fp.idx:n]
		term_len = len(term)
		n = skip_white(fp.line, n)
		if n < fp.line_len and fp.line[n] != '#':
		    recipe_error(fp.rpstack,
					   _("Too many arguments for :python"))
	    start_line_nr = fp.rpstack[-1].line_nr
	    first = 1
	    while 1:
		fp.nextline()
		if fp.line is None:
		    if not term:
			break
		    fp.rpstack[-1].line_nr = start_line_nr
		    recipe_error(fp.rpstack, _("Unterminated :python block"))

		if first:
		    first = 0
		    # If the indent of the Python block is more than the
		    # current indent, insert an ":if 1".
		    if get_indent(fp.line) > indent:
			script = script + (indent * ' ') + "if 1:" + '\n'

		if not term:
		    # No terminator defined: end when indent is smaller.
		    if get_indent(fp.line) <= indent:
			break
		else:
		    # Terminator defined: end when it's found.
		    n = skip_white(fp.line, 0)
		    if n < fp.line_len and fp.line[n:n + term_len] == term:
			n = skip_white(fp.line, n + term_len)
			if n >= fp.line_len or fp.line[n] == "#":
			    fp.nextline()
			    break

		# Append the recipe line number, used for error messages.
		script = script + ("%s%d\n%s\n"
			      % (line_marker, fp.rpstack[-1].line_nr, fp.line))
	    continue

	#
	# An A-A-P command
	#
	#  recipe:  :cmd arg arg
	#                  arg
	#  Python:  aap_cmd(123, globals(), "arg arg arg")
	#
	if fp.line[fp.idx] == ":":
	    s = fp.idx
	    fp.idx = fp.idx + 1
	    e = skip_to_white(fp.line, fp.idx)
	    cmd_name = fp.line[fp.idx:e]
	    fp.idx = skip_white(fp.line, e)

	    # Check if this is a valid command name.  The error is postponed
	    # until executing the line, so that "@if aapversion > nr" can be
	    # used before it.
	    if cmd_name not in aap_cmd_names:
		cmd_name = "unknown"
		fp.idx = s
	    if fp.string and cmd_name in aap_cmd_toplevel:
		cmd_name = "nothere"
		fp.idx = s

	    #
	    # To avoid starting a shell for every single command, collect
	    # system commands until encountering another command.
	    #
	    # recipe:       :system one-shell-command
	    #		    :sys  two-shell-command
	    # Python:	    aap_shell(123, globals(),
	    #			    "one-shell_command\ntwo_shell_command\n")
	    #
	    if cmd_name == "system" or cmd_name == "sys":
		if not shell_cmd:
		    shell_cmd_line_nr = fp.rpstack[-1].line_nr
		    shell_cmd_indent = indent
		shell_cmd = shell_cmd + getarg(fp, "#", globals) + '\\n'

		# get the next line
		fp.nextline()
		continue

	    # recipe:   :variant VAR
	    #               foo  [ condition ]
	    #                  cmds
	    #               *    [ condition ]
	    #                  cmds
	    # Python:   if VAR == "foo":
	    #                  cmds
	    #           else:
	    #                  cmds
	    #           BDIR = BDIR + '-' + VAR
	    # This is complicated, because "cmds" can be any command, and
	    # variants may nest.  Store the info about the variant in
	    # variant_stack and continue, the rest is handled above.
	    if cmd_name == "variant":
		var_name, n = get_var_name(fp)
		if var_name == '' or (n < fp.line_len and fp.line[n] != '#'):
		    recipe_error(fp.rpstack,
				   _("Expected variable name after :variant"))
		variant_stack.append(Variant(var_name, indent))

		# get the next line
		fp.nextline()
		continue

	    # Generate a call to the Python function for this command.
	    script = script + (indent * ' ') + ('aap_%s(%d, globals(), '
					  % (cmd_name, fp.rpstack[-1].line_nr))

	    if cmd_name == "rule" or cmd_name == "autodepend":
		# recipe:   :rule  target : {attr} source
		#                 commands
		# Python:   aap_rule(123, globals(), "target", "source",
		#                      124, """commands""")
		#
		# recipe:   :autodepend  {attr} source
		#                 commands
		# Python:   aap_autodepend(123, globals(), "source",
		#                      124, """commands""")
		if cmd_name == "rule":
		    target = getarg(fp, ":#", globals)
		    if fp.idx >= fp.line_len or fp.line[fp.idx] != ':':
			recipe_error(fp.rpstack, _("Missing ':' after :%s")
								    % cmd_name)
		    fp.idx = fp.idx + 1
		    script = script + ('"%s", ' % target)

		source = getarg(fp, "#", globals)

		cmd_line_nr = fp.rpstack[-1].line_nr
		fp.nextline()
		cmds = get_commands(fp, indent)

		script = script + ('"%s", %d, %s)\n'
						 % (source, cmd_line_nr, cmds))

	    elif cmd_name == "filetype":
		# recipe:   :filetype [filename]
		#		detection-lines
		# Python:   aap_filetype_python(123, globals(), "arg",
		#				    """detection-lines""")
		#
		arg = getarg(fp, "#", globals)
		cmd_line_nr = fp.rpstack[-1].line_nr
		fp.nextline()
		cmds = get_commands(fp, indent)
		script = script + '"%s", %d, %s)\n' % (arg, cmd_line_nr, cmds)

	    else:
		# get arguments that may continue on the next line
		script = script + get_func_args(fp, indent, globals) + ")\n"

		# When a ":recipe" command is encountered that will probably
		# be executed, make a copy of the globals at the start, so that
		# this can be restored when executing the updated recipe.
		# This is a "heavy" command, only do it when needed.
		from Commands import do_recipe_cmd
		if (cmd_name == "recipe" and not had_recipe_cmd
						and do_recipe_cmd(fp.rpstack)):
		    had_recipe_cmd = 1
		    script = ('globals()["_start_globals"] = globals().copy()\n'
			    + script)

	    continue

	#
	# A Python command
	#
	#  recipe:    @command args
	#  Python:	  command args
	#
	if fp.line[fp.idx] == "@":
	    if fp.idx + 1 < fp.line_len:
		if fp.line[fp.idx + 1] == ' ' \
					    or fp.line[fp.idx + 1] == '\t':
		    # followed by white space: replace @ with a space
		    script = script \
			      + string.replace(fp.line, '@', ' ', 1) + '\n'
		else:
		    # followed by text: remove the @
		    script = script \
			       + string.replace(fp.line, '@', '', 1) + '\n'

	    # get the next line
	    fp.nextline()
	    continue

	#
	# Assignment
	#
	#  recipe:   name = $VAR {attr=val} ` glob("*.c") `
	#                     two
	#  Python:	 name = aap_eval(123, globals(),
	#			"$VAR {attr=val} " + glob("*.c") + " two", 1)
	#
	#  var = value	    assign
	#  var += value	    append (assign if not set yet)
	#  var ?= value	    only assign when not set yet
	#  var $= value	    evaluate when used
	#  var $+= value    append, evaluate when used
	#  var $?= value    only when not set, evaluate when used
	var_name, n = get_var_name(fp)

	if n < fp.line_len:
	    nc = fp.line[n]
	    ec = nc
	    if ec == '$' and n + 1 < fp.line_len:
		ne = n + 1
		ec = fp.line[ne]
	    else:
		ne = n
	    if (ec == '+' or ec == '?') and ne + 1 < fp.line_len:
		lc = ec
		ne = ne + 1
		ec = fp.line[ne]
	    else:
		lc = ''
	    if var_name != '' and ec == '=':
		# When changing $CACHE need to flush the cache and reload it.
		if var_name == "CACHE":
		    script = script + (indent * ' ') + "flush_cache()\n"
		fp.idx = skip_white(fp.line, ne + 1)
		script = script + (indent * ' ') + (
				     "aap_assign(%d, globals(), '%s', "
					  % (fp.rpstack[-1].line_nr, var_name))
		args = get_func_args(fp, indent, globals)
		script = script + args + (", '%s', '%s')\n" % (nc, lc))
		continue

	#
	# If there is no ":" following we don't know what it is.
	#
	targets = getarg(fp, ":#", globals)
	if fp.idx >= fp.line_len or fp.line[fp.idx] != ':':
	    recipe_error(fp.rpstack, _("No recognized item"))

	if fp.string:
	    recipe_error(fp.rpstack, _("Dependency not allowed here"))

	#
	# Dependency
	#
	#  recipe:     target target : source source
	#                    commands
	#  Python:     aap_depend(123, globals(), list-of-targets,
	#			      list-of-sources, "commands")
	#
	else:
	    # Skip the ':' and get the list of sources.
	    fp.idx = skip_white(fp.line, fp.idx + 1)
	    sources = getarg(fp, '#', globals)
	    nr = fp.rpstack[-1].line_nr
	    fp.nextline()
	    script = script + ('aap_depend(%d, globals(), "%s", "%s", %d, '
			      % (nr, targets, sources, fp.rpstack[-1].line_nr))

	    # get the commands and the following line
	    cmds = get_commands(fp, indent)
	    script = script + cmds + ')\n'

	#
	# End of loop over all lines in recipe.
	#


    if fp.string:
	# When parsing a string need to take care of the indent.
	if is_white(fp.string[0]):
	    script = "if 1:\n" + script
    else:
	# Close the file before executing the script, so that ":recipe" can
	# overwrite the file.
	fp.file.close()

    # Prepend the default imports.
    script = "from Commands import *\n" \
		+ "from glob import glob\n" \
		+ script
    
    # DEBUG
    #print script

    #
    # Execute the resulting Python script.
    # Give a useful error message when something is wrong.
    #
    try:
	exec script in globals, globals
    except StandardError, e:
	script_error(fp.rpstack, script, e)


# vim: set sw=4 sts=4 tw=79 fo+=l: