add a check to support errors being raised before tempfiles are created

minerva/athena.py

@@ -112,8 +112,6 @@ class Query(Execute):
             files = self.handler.s3.read(manif).split("\n")
             files = [f.strip() for f in files if f.strip()] # filter empty
 
-            self.files = files
-
             return files
         else:
             print("Error")
@@ -125,11 +123,9 @@ class Query(Execute):
     # dataset of the results.
     # Calls `self.manifest_files()` which blocks via `self.finish()`
     def results(self):
-        start   = time.time()
-        local   = [self.handler.s3.download(f) for f in self.manifest_files()]
-        #local   = parallel_map(self.handler.s3.download, self.manifest_files())
-        print(time.time() - start)
-        self.ds = pa.dataset.dataset(local)
+        self.temps = [self.handler.s3.download(f) for f in self.manifest_files()]
+        #local      = parallel_map(self.handler.s3.download, self.manifest_files())
+        self.ds = pa.dataset.dataset(self.temps)
         return self.ds
 
     def __enter__(self):
@@ -139,6 +135,7 @@ class Query(Execute):
         self.close()
 
     def close(self):
-        for file in self.files:
-            os.remove(file)
+        if self.temps:
+            for file in self.temps:
+                os.remove(file)
 

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "minerva"
-version = "0.3.4"
+version = "0.3.5"
 description = "Easier access to AWS Athena and Redshift"
 authors = [
   "Ari Brown <ari@airintech.com>",