ari/minerva
1
0
Fork 0
forked from bellwether/minerva
Code Pull requests Activity

fixed dataset aggregation; typo; and a bad empty string check

Browse source
This commit is contained in:
Ari Brown 2024-01-26 11:48:35 -05:00
parent a907be22cc
commit 819bf7abf3
2 changed files with 7 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

3
examples/cancel_queries.py
View file

@ -10,6 +10,9 @@ with open(file, 'r') as f:
txt = f.read() txt = f.read()
for line in txt.split("\n"): for line in txt.split("\n"):
if not line.strip():
continue
print(line) print(line)
athena.cancel(line) athena.cancel(line)

8
minerva/athena.py
View file

@ -282,7 +282,7 @@ class Parallelize:
def results(self): def results(self):
self.finish() self.finish()
return pa.dataset.dataset([q.results().files for q in self.queries]) return pa.dataset.dataset([f for q in self.queries for f in q.results().files])
def union_tables(self, dest): def union_tables(self, dest):
@ -292,8 +292,8 @@ class Parallelize:
tables = ' union all '.join(lines) tables = ' union all '.join(lines)
out = os.path.join(self.athena.output, dest) out = os.path.join(self.athena.output, dest)
sql = load_sql(self.UNION_TABLES, dest = dest, sql = load_template(self.UNION_TABLES, dest = dest,
output = out, output = out,
tables = tables) tables = tables)
return self.athena.execute(sql) return self.athena.execute(sql)