ari/minerva
1
0
Fork 0
forked from bellwether/minerva
Code Pull requests Activity

added binary file for an easy athena console

Browse source
This commit is contained in:
Ari Brown 2023-09-15 10:20:53 -04:00
parent 383185e6cb
commit ab344374d9
3 changed files with 50 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

13
minerva/athena.py
View file

@ -42,6 +42,7 @@ class Execute:
self.params = [str(p) for p in params]
self.info_cache = None
self.temps = []
self.ds = None
# The string of the query
def query(self):
@ -80,6 +81,10 @@ class Execute:
ms = self.info_cache['Statistics']['TotalExecutionTimeInMillis']
self.runtime = datetime.timedelta(seconds=ms / 1000)
scanned = self.info_cache['Statistics']['DataScannedInBytes']
self.cost = 5.0 * scanned / (1024 ** 4) # $5/TB scanned
return stat # finalized state
@ -122,11 +127,19 @@ class Query(Execute):
# dataset of the results.
# Calls `self.manifest_files()` which blocks via `self.finish()`
def results(self):
if self.ds:
return self.ds
self.temps = [self.handler.s3.download(f) for f in self.manifest_files()]
#local = parallel_map(self.handler.s3.download, self.manifest_files())
self.ds = pa.dataset.dataset(self.temps)
return self.ds
# Return scalar results
# Abstracts away a bunch of keystrokes
def scalar(self):
return self.results().head(1)[0][0].as_py()
def __enter__(self):
return self