Index: /branches/fc15-dev/host/credit-card/host.py =================================================================== --- /branches/fc15-dev/host/credit-card/host.py (revision 1999) +++ /branches/fc15-dev/host/credit-card/host.py (revision 1999) @@ -0,0 +1,147 @@ +import os +import optparse +import logging +import socket +import tempfile +import shutil +import errno + +import shell + +HOST = socket.gethostname() + +# XXX test server and wizard server + +ROOT_UID = 0 +SIGNUP_UID = 102 +SQL_UID = 537704221 +FEDORA_DS_UID = 103 # XXX ACTUALLY CONFIGURE SERVERS TO USE THIS +LOGVIEW_UID = 501 # XXX Autogenerated, I don't like this... + +COMMON_CREDS = [ + (ROOT_UID, 0o600, 'root/.bashrc'), + (ROOT_UID, 0o600, 'root/.screenrc'), + (ROOT_UID, 0o600, 'root/.ssh/authorized_keys'), + (ROOT_UID, 0o600, 'root/.ssh/authorized_keys2'), + (ROOT_UID, 0o600, 'root/.vimrc'), + (ROOT_UID, 0o600, 'root/.k5login'), + # punted /root/.ssh/known_hosts + + # XXX must be created in Kickstart + (LOGVIEW_UID, 0o600, 'home/logview/.k5login'), + ] + +COMMON_PROD_CREDS = [ # important: no leading slashes! + (ROOT_UID, 0o600, 'root/.ldapvirc'), + (ROOT_UID, 0o600, 'etc/ssh/ssh_host_dsa_key'), + (ROOT_UID, 0o600, 'etc/ssh/ssh_host_key'), + (ROOT_UID, 0o600, 'etc/ssh/ssh_host_rsa_key'), + (ROOT_UID, 0o600, 'etc/pki/tls/private/scripts.key'), + (ROOT_UID, 0o600, 'etc/whoisd-password'), + (ROOT_UID, 0o600, 'etc/daemon.keytab'), + + (ROOT_UID, 0o644, 'etc/ssh/ssh_host_dsa_key.pub'), + (ROOT_UID, 0o644, 'etc/ssh/ssh_host_key.pub'), + (ROOT_UID, 0o644, 'etc/ssh/ssh_host_rsa_key.pub'), + + (SQL_UID, 0o600, 'etc/sql-mit-edu.cfg.php'), + (SIGNUP_UID, 0o600, 'etc/signup-ldap-pw'), + ] + +MACHINE_PROD_CREDS = [ + # XXX NEED TO CHECK THAT THESE ARE SENSIBLE + (ROOT_UID, 0o600, 'etc/krb5.keytab'), + (FEDORA_DS_UID, 0o600, 'etc/dirsrv/keytab') + ] + +def mkdir_p(path): + try: + os.makedirs(path) + except OSError as exc: # Python >2.5 + if exc.errno == errno.EEXIST: + pass + else: raise + +class WithMount(object): + """Context for running code with an extra mountpoint.""" + guest = None + mount = None + dev = None + def __init__(self, guest): + self.guest = guest + def __enter__(self): + self.dev = "/dev/%s/%s-root" % (HOST, self.guest) + + mapper_name = shell.eval("kpartx", "-l", self.dev).split()[0] + shell.call("kpartx", "-a", self.dev) + mapper = "/dev/mapper/%s" % mapper_name + + # this is why bracketing functions and hanging lambdas are a good idea + try: + self.mount = tempfile.mkdtemp("-%s" % self.guest, 'vm-', '/mnt') # no trailing slash + try: + shell.call("mount", mapper, self.mount) + except: + os.rmdir(self.mount) + raise + except: + shell.call("kpartx", "-d", self.dev) + raise + + return self.mount + def __exit__(self, *args): + shell.call("umount", self.mount) + os.rmdir(self.mount) + shell.call("kpartx", "-d", self.dev) + +def main(): + usage = """usage: %prog [ARGS]""" + + parser = optparse.OptionParser(usage) + _, args = parser.parse_args() + + creds = "/root/creds" # XXX check exists, check owned by root + # make an option + if not os.path.isdir(creds): + raise Exception("/root/creds does not exist") + + os.umask(0o077) # overly restrictive + + # XXX error handling + + if len(args) != 2: + raise Exception("Wrong number of arguments") + + command = args[0] + guest = args[1] + + with WithMount(guest) as tmp_mount: + def push_files(files, type): + for (ugid, perms, f) in files: + # assumes directories exist + dest = "%s/%s" % (tmp_mount, f) + # assuming OK to overwrite + shutil.copyfile("%s/%s/%s" % (creds, type, f), dest) + os.chown(dest, ugid, ugid) + os.chmod(dest, perms) + def pull_files(files, type): + for (_, _, f) in files: + dest = "%s/%s/%s" % (creds, type, f) + mkdir_p(os.path.dirname(dest)) + # error if doesn't exist + shutil.copyfile("%s/%s" % (tmp_mount, f), dest) + + # push case + if command == "push": + push_files(COMMON_CREDS, 'common') + push_files(COMMON_PROD_CREDS, 'common') + push_files(MACHINE_PROD_CREDS, 'machine/%s' % guest) + elif command == "pull": + # check if /root/creds exists + pull_files(MACHINE_PROD_CREDS, 'machine/%s' % guest) + elif command == "pull-common": + pull_files(COMMON_CREDS, 'common') + pull_files(COMMON_PROD_CREDS, 'common') + +if __name__ == "__main__": + main() Index: /branches/fc15-dev/host/credit-card/shell.py =================================================================== --- /branches/fc15-dev/host/credit-card/shell.py (revision 1999) +++ /branches/fc15-dev/host/credit-card/shell.py (revision 1999) @@ -0,0 +1,301 @@ +""" +Wrappers around subprocess functionality that simulate an actual shell. +""" + +import subprocess +import logging +import sys +import os +import errno + +class Shell(object): + """ + An advanced shell that performs logging. If ``dry`` is ``True``, + no commands are actually run. + """ + def __init__(self, dry = False): + self.dry = dry + self.cwd = None + def call(self, *args, **kwargs): + """ + Performs a system call. The actual executable and options should + be passed as arguments to this function. Several keyword arguments + are also supported: + + :param input: input to feed the subprocess on standard input. + :param interactive: whether or not directly hook up all pipes + to the controlling terminal, to allow interaction with subprocess. + :param strip: if ``True``, instead of returning a tuple, + return the string stdout output of the command with trailing newlines + removed. This emulates the behavior of backticks and ``$()`` in Bash. + Prefer to use :meth:`eval` instead (you should only need to explicitly + specify this if you are using another wrapper around this function). + :param log: if True, we log the call as INFO, if False, we log the call + as DEBUG, otherwise, we detect based on ``strip``. + :param stdout: + :param stderr: + :param stdin: a file-type object that will be written to or read from as a pipe. + :returns: a tuple of strings ``(stdout, stderr)``, or a string ``stdout`` + if ``strip`` is specified. + + >>> sh = Shell() + >>> sh.call("echo", "Foobar") + ('Foobar\\n', '') + >>> sh.call("cat", input='Foobar') + ('Foobar', '') + """ + self._wait() + kwargs.setdefault("interactive", False) + kwargs.setdefault("strip", False) + kwargs.setdefault("python", None) + kwargs.setdefault("log", None) + kwargs.setdefault("stdout", subprocess.PIPE) + kwargs.setdefault("stdin", subprocess.PIPE) + kwargs.setdefault("stderr", subprocess.PIPE) + msg = "Running `" + ' '.join(args) + "`" + if kwargs["strip"] and not kwargs["log"] is True or kwargs["log"] is False: + logging.debug(msg) + else: + logging.info(msg) + if self.dry: + if kwargs["strip"]: + return '' + return None, None + kwargs.setdefault("input", None) + if kwargs["interactive"]: + stdout=sys.stdout + stdin=sys.stdin + stderr=sys.stderr + else: + stdout=kwargs["stdout"] + stdin=kwargs["stdin"] + stderr=kwargs["stderr"] + # XXX: There is a possible problem here where we can fill up + # the kernel buffer if we have 64KB of data. This shouldn't + # be a problem, and the fix for such case would be to write to + # temporary files instead of a pipe. + # Another possible way of fixing this is converting from a + # waitpid() pump to a select() pump, creating a pipe to + # ourself, and then setting up a + # SIGCHILD handler to write a single byte to the pipe to get + # us out of select() when a subprocess exits. + proc = subprocess.Popen(args, stdout=stdout, stderr=stderr, stdin=stdin, cwd=self.cwd, ) + if self._async(proc, args, **kwargs): + return proc + stdout, stderr = proc.communicate(kwargs["input"]) + # can occur if we were doing interactive communication; i.e. + # we didn't pass in PIPE. + if stdout is None: + stdout = "" + if stderr is None: + stderr = "" + if not kwargs["interactive"]: + if kwargs["strip"]: + self._log(None, stderr) + else: + self._log(stdout, stderr) + if proc.returncode: + raise CallError(proc.returncode, args, stdout, stderr) + if kwargs["strip"]: + return str(stdout).rstrip("\n") + return (stdout, stderr) + def _log(self, stdout, stderr): + """Logs the standard output and standard input from a command.""" + if stdout: + logging.debug("STDOUT:\n" + stdout) + if stderr: + logging.debug("STDERR:\n" + stderr) + def _wait(self): + pass + def _async(self, *args, **kwargs): + return False + def callAsUser(self, *args, **kwargs): + """ + Performs a system call as a different user. This is only possible + if you are running as root. Keyword arguments + are the same as :meth:`call` with the following additions: + + :param user: name of the user to run command as. + :param uid: uid of the user to run command as. + + .. note:: + + The resulting system call internally uses :command:`sudo`, + and as such environment variables will get scrubbed. We + manually preserve :envvar:`SSH_GSSAPI_NAME`. + """ + user = kwargs.pop("user", None) + uid = kwargs.pop("uid", None) + if not user and not uid: return self.call(*args, **kwargs) + if os.getenv("SSH_GSSAPI_NAME"): + # This might be generalized as "preserve some environment" + args = list(args) + args.insert(0, "SSH_GSSAPI_NAME=" + os.getenv("SSH_GSSAPI_NAME")) + if uid: return self.call("sudo", "-u", "#" + str(uid), *args, **kwargs) + if user: return self.call("sudo", "-u", user, *args, **kwargs) + def safeCall(self, *args, **kwargs): + """ + Checks if the owner of the current working directory is the same + as the current user, and if it isn't, attempts to sudo to be + that user. The intended use case is for calling Git commands + when running as root, but this method should be used when + interfacing with any moderately complex program that depends + on working directory context. Keyword arguments are the + same as :meth:`call`. + """ + if os.getuid(): + return self.call(*args, **kwargs) + uid = os.stat(os.getcwd()).st_uid + # consider also checking ruid? + if uid != os.geteuid(): + kwargs['uid'] = uid + return self.callAsUser(*args, **kwargs) + else: + return self.call(*args, **kwargs) + def eval(self, *args, **kwargs): + """ + Evaluates a command and returns its output, with trailing newlines + stripped (like backticks in Bash). This is a convenience method for + calling :meth:`call` with ``strip``. + + >>> sh = Shell() + >>> sh.eval("echo", "Foobar") + 'Foobar' + """ + kwargs["strip"] = True + return self.call(*args, **kwargs) + def setcwd(self, cwd): + """ + Sets the directory processes are executed in. This sets a value + to be passed as the ``cwd`` argument to ``subprocess.Popen``. + """ + self.cwd = cwd + +class ParallelShell(Shell): + """ + Modifies the semantics of :class:`Shell` so that + commands are queued here, and executed in parallel using waitpid + with ``max`` subprocesses, and result in callback execution + when they finish. + + .. method:: call(*args, **kwargs) + + Enqueues a system call for parallel processing. If there are + no openings in the queue, this will block. Keyword arguments + are the same as :meth:`Shell.call` with the following additions: + + :param on_success: Callback function for success (zero exit status). + The callback function should accept two arguments, + ``stdout`` and ``stderr``. + :param on_error: Callback function for failure (nonzero exit status). + The callback function should accept one argument, the + exception that would have been thrown by the synchronous + version. + :return: The :class:`subprocess.Proc` object that was opened. + + .. method:: callAsUser(*args, **kwargs) + + Enqueues a system call under a different user for parallel + processing. Keyword arguments are the same as + :meth:`Shell.callAsUser` with the additions of keyword + arguments from :meth:`call`. + + .. method:: safeCall(*args, **kwargs) + + Enqueues a "safe" call for parallel processing. Keyword + arguments are the same as :meth:`Shell.safeCall` with the + additions of keyword arguments from :meth:`call`. + + .. method:: eval(*args, **kwargs) + + No difference from :meth:`call`. Consider having a + non-parallel shell if the program you are shelling out + to is fast. + + """ + def __init__(self, dry = False, max = 10): + super(ParallelShell, self).__init__(dry=dry) + self.running = {} + self.max = max # maximum of commands to run in parallel + @staticmethod + def make(no_parallelize, max): + """Convenience method oriented towards command modules.""" + if no_parallelize: + return DummyParallelShell() + else: + return ParallelShell(max=max) + def _async(self, proc, args, python, on_success, on_error, **kwargs): + """ + Gets handed a :class:`subprocess.Proc` object from our deferred + execution. See :meth:`Shell.call` source code for details. + """ + self.running[proc.pid] = (proc, args, python, on_success, on_error) + return True # so that the parent function returns + def _wait(self): + """ + Blocking call that waits for an open subprocess slot. This is + automatically called by :meth:`Shell.call`. + """ + # XXX: This API sucks; the actual call/callAsUser call should + # probably block automatically (unless I have a good reason not to) + # bail out immediately on initial ramp up + if len(self.running) < self.max: return + # now, wait for open pids. + try: + self.reap(*os.waitpid(-1, 0)) + except OSError as e: + if e.errno == errno.ECHILD: return + raise + def join(self): + """Waits for all of our subprocesses to terminate.""" + try: + while True: + self.reap(*os.waitpid(-1, 0)) + except OSError as e: + if e.errno == errno.ECHILD: return + raise + def reap(self, pid, status): + """Reaps a process.""" + # ooh, zombie process. reap it + proc, args, python, on_success, on_error = self.running.pop(pid) + # XXX: this is slightly dangerous; should actually use + # temporary files + stdout = proc.stdout.read() + stderr = proc.stderr.read() + self._log(stdout, stderr) + if status: + on_error(CallError(proc.returncode, args, stdout, stderr)) + return + on_success(stdout, stderr) + +# Setup a convenience global instance +shell = Shell() +call = shell.call +callAsUser = shell.callAsUser +safeCall = shell.safeCall +eval = shell.eval + +class DummyParallelShell(ParallelShell): + """Same API as :class:`ParallelShell`, but doesn't actually + parallelize (i.e. all calls to :meth:`wait` block.)""" + def __init__(self, dry = False): + super(DummyParallelShell, self).__init__(dry=dry, max=1) + +class CallError: + """Indicates that a subprocess call returned a nonzero exit status.""" + #: The exit code of the failed subprocess. + code = None + #: List of the program and arguments that failed. + args = None + #: The stdout of the program. + stdout = None + #: The stderr of the program. + stderr = None + def __init__(self, code, args, stdout, stderr): + self.code = code + self.args = args + self.stdout = stdout + self.stderr = stderr + def __str__(self): + compact = self.stderr.rstrip().split("\n")[-1] + return "%s (exited with %d)\n%s" % (compact, self.code, self.stderr)