diff --git a/minerva/docker.py b/minerva/docker.py index 97eefc4..b2b7ab3 100644 --- a/minerva/docker.py +++ b/minerva/docker.py @@ -46,15 +46,15 @@ class Docker: res = self.machine.docker_run(self.uri, cmd=cmd, env=self.variables) - self.out["stdout"] = res.stdout - self.out["stderr"] = res.stderr + self.out["stdout"] = res[0].read + self.out["stderr"] = res[1].read if self.stdout: - self.stdout.write(res.stdout) + self.stdout.write(res[0].read) self.stdout.write("\n") if self.stderr: - self.stderr.write(res.stderr) + self.stderr.write(res[1].read) self.stderr.write("\n") self.finished = True diff --git a/minerva/remote.py b/minerva/remote.py index 204cfc2..a420372 100644 --- a/minerva/remote.py +++ b/minerva/remote.py @@ -2,6 +2,8 @@ from fabric import Connection import os import threading import select +import tempfile +import io # Bare machine, not necessarily associated with AWS class Remote: @@ -41,21 +43,18 @@ class Remote: # `disown` means it'll run in the background # # TODO switch this to use `self.ssh.client.exec_command()` - def cmd(self, command, hide=True, disown=False, watch=False): - res = self.ssh.run(f"{self.prep_variables()}; {command}", - warn=True, - hide=hide, - disown=disown) + #def cmd(self, command, hide=True, disown=False, watch=False): + # res = self.ssh.run(f"{self.prep_variables()}; {command}", + # warn=True, + # hide=hide, + # disown=disown) - return res + # return res # https://github.com/paramiko/paramiko/issues/593#issuecomment-145377328 # https://stackoverflow.com/questions/23504126/do-you-have-to-check-exit-status-ready-if-you-are-going-to-check-recv-ready/32758464#32758464 # def cmd(self, command, hide=True, disown=False, watch=False): - print(self.ssh) - print(self.ssh.client) - self.ssh.run("echo hello world", warn=True, hide=hide, disown=disown) stdin, stdout, stderr = self.ssh.client.exec_command(command) @@ -63,12 +62,21 @@ class Remote: # this is the same for all three inputs channel = stdin.channel - out_r, out_w = os.pipe() - err_r, err_w = os.pipe() - - os.write(out_w, b"hello world") + # regular TemporaryFile doesn't work for some reason, even with + # explicit flush(). I think it's because it doesn't actually create + # a file on disk until enough input has been gathered. + # + # A flush is required after every write + # Leave the files so that the readers can work even after the writers + # are done + # + # Thanks to SirDonNick in #python for the help here + out = tempfile.NamedTemporaryFile(delete=False) + err = tempfile.NamedTemporaryFile(delete=False) + # Taken from # https://stackoverflow.com/a/78765054 + # and then improved/cleaned up # we do not need stdin. stdin.close() @@ -76,9 +84,8 @@ class Remote: channel.shutdown_write() # read stdout/stderr to prevent read block hangs - stdout_chunks = [] - stderr_chunks = [] - stdout_chunks.append(channel.recv(len(channel.in_buffer))) + out.write(channel.recv(len(channel.in_buffer))) + err.write(channel.recv_stderr(len(channel.in_stderr_buffer))) timeout = 60 @@ -96,20 +103,12 @@ class Remote: break for c in readq: if c.recv_ready(): - #stdout_chunks.append(channel.recv(len(c.in_buffer))) - data = channel.recv(len(c.in_buffer)) - if data: - print("*******") - print(repr(data)) - os.write(out, data) + out.write(channel.recv(len(c.in_buffer))) + out.flush() got_chunk = True if c.recv_stderr_ready(): - #stderr_chunks.append(channel.recv_stderr(len(c.in_stderr_buffer))) - data = channel.recv_stderr(len(c.in_stderr_buffer)) - if data: - print("*******") - print(repr(data)) - os.write(err, data) + err.write(channel.recv_stderr(len(c.in_stderr_buffer))) + err.flush() got_chunk = True # for c @@ -131,24 +130,23 @@ class Remote: # remote side is finished and our buffers are empty break # if + out.close() + err.close() # while # close the pseudofiles stdout.close() stderr.close() - #thread = threading.Thread(target = fill_buffers, - # args = (out_w, err_w)) - #thread.start() - fill_buffers(out_w, err_w) + thread = threading.Thread(target = fill_buffers, + args = (out, err)) + thread.start() - #if not disown: - # thread.join() + if not disown: + print("joining") + thread.join() - #return (stdout_chunks, stderr_chunks) - os.close(out_w) - os.close(err_w) - return (os.fdopen(out_r), os.fdopen(err_r)), thread) + return (open(out.name, "rb"), open(err.name, "rb"), thread) def write_env_file(self, variables, fname="~/env.list"):