whew, got partial reads working with named temporary files. lots of weird oddities were discovered in the process

minerva/docker.py

@@ -46,15 +46,15 @@ class Docker:
 
         res = self.machine.docker_run(self.uri, cmd=cmd, env=self.variables)
 
-        self.out["stdout"] = res.stdout
+        self.out["stdout"] = res[0].read
-        self.out["stderr"] = res.stderr
+        self.out["stderr"] = res[1].read
 
         if self.stdout:
-            self.stdout.write(res.stdout)
+            self.stdout.write(res[0].read)
             self.stdout.write("\n")
 
         if self.stderr:
-            self.stderr.write(res.stderr)
+            self.stderr.write(res[1].read)
             self.stderr.write("\n")
 
         self.finished = True

minerva/remote.py

@@ -2,6 +2,8 @@ from fabric import Connection
 import os
 import threading
 import select
+import tempfile
+import io
 
 # Bare machine, not necessarily associated with AWS
 class Remote:
@@ -41,21 +43,18 @@ class Remote:
     # `disown` means it'll run in the background
     #
     # TODO switch this to use `self.ssh.client.exec_command()`
-    def cmd(self, command, hide=True, disown=False, watch=False):
+    #def cmd(self, command, hide=True, disown=False, watch=False):
-        res = self.ssh.run(f"{self.prep_variables()}; {command}",
+    #    res = self.ssh.run(f"{self.prep_variables()}; {command}",
-                           warn=True,
+    #                       warn=True,
-                           hide=hide,
+    #                       hide=hide,
-                           disown=disown)
+    #                       disown=disown)
 
-        return res
+    #    return res
 
     # https://github.com/paramiko/paramiko/issues/593#issuecomment-145377328
     # https://stackoverflow.com/questions/23504126/do-you-have-to-check-exit-status-ready-if-you-are-going-to-check-recv-ready/32758464#32758464
     #
     def cmd(self, command, hide=True, disown=False, watch=False):
-        print(self.ssh)
-        print(self.ssh.client)
-
         self.ssh.run("echo hello world", warn=True, hide=hide, disown=disown)
 
         stdin, stdout, stderr = self.ssh.client.exec_command(command)
@@ -63,12 +62,21 @@ class Remote:
         # this is the same for all three inputs
         channel = stdin.channel
 
-        out_r, out_w = os.pipe()
+        # regular TemporaryFile doesn't work for some reason, even with
-        err_r, err_w = os.pipe()
+        # explicit flush(). I think it's because it doesn't actually create
-
+        # a file on disk until enough input has been gathered.
-        os.write(out_w, b"hello world")
+        #
+        # A flush is required after every write
+        # Leave the files so that the readers can work even after the writers
+        # are done
+        #
+        # Thanks to SirDonNick in #python for the help here
+        out = tempfile.NamedTemporaryFile(delete=False)
+        err = tempfile.NamedTemporaryFile(delete=False)
 
+        # Taken from
         # https://stackoverflow.com/a/78765054
+        # and then improved/cleaned up
 
         # we do not need stdin.
         stdin.close()
@@ -76,9 +84,8 @@ class Remote:
         channel.shutdown_write()
 
         # read stdout/stderr to prevent read block hangs
-        stdout_chunks = []
+        out.write(channel.recv(len(channel.in_buffer)))
-        stderr_chunks = []
+        err.write(channel.recv_stderr(len(channel.in_stderr_buffer)))
-        stdout_chunks.append(channel.recv(len(channel.in_buffer)))
 
         timeout = 60
 
@@ -96,20 +103,12 @@ class Remote:
                     break
                 for c in readq:
                     if c.recv_ready():
-                        #stdout_chunks.append(channel.recv(len(c.in_buffer)))
+                        out.write(channel.recv(len(c.in_buffer)))
-                        data = channel.recv(len(c.in_buffer))
+                        out.flush()
-                        if data:
-                            print("*******")
-                            print(repr(data))
-                            os.write(out, data)
                         got_chunk = True
                     if c.recv_stderr_ready():
-                        #stderr_chunks.append(channel.recv_stderr(len(c.in_stderr_buffer)))
+                        err.write(channel.recv_stderr(len(c.in_stderr_buffer)))
-                        data = channel.recv_stderr(len(c.in_stderr_buffer))
+                        err.flush()
-                        if data:
-                            print("*******")
-                            print(repr(data))
-                            os.write(err, data)
                         got_chunk = True
                 # for c
 
@@ -131,24 +130,23 @@ class Remote:
                     # remote side is finished and our buffers are empty
                     break
                     # if
+            out.close()
+            err.close()
             # while
 
             # close the pseudofiles
             stdout.close()
             stderr.close()
 
-        #thread = threading.Thread(target = fill_buffers,
+        thread = threading.Thread(target = fill_buffers,
-        #                          args   = (out_w, err_w))
+                                  args   = (out, err))
-        #thread.start()
+        thread.start()
-        fill_buffers(out_w, err_w)
 
-        #if not disown:
+        if not disown:
-        #    thread.join()
+            print("joining")
+            thread.join()
 
-        #return (stdout_chunks, stderr_chunks)
+        return (open(out.name, "rb"), open(err.name, "rb"), thread)
-        os.close(out_w)
-        os.close(err_w)
-        return (os.fdopen(out_r), os.fdopen(err_r)), thread)
 
 
     def write_env_file(self, variables, fname="~/env.list"):