ari/minerva
1
0
Fork 0
forked from bellwether/minerva
Code Pull requests Activity

fixed dataset aggregation; typo; and a bad empty string check

Browse source
This commit is contained in:
Ari Brown 2024-01-26 11:48:35 -05:00
parent a907be22cc
commit 819bf7abf3
2 changed files with 7 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

8
minerva/athena.py
View file

@ -282,7 +282,7 @@ class Parallelize:
def results(self):
self.finish()
return pa.dataset.dataset([q.results().files for q in self.queries])
return pa.dataset.dataset([f for q in self.queries for f in q.results().files])
def union_tables(self, dest):
@ -292,8 +292,8 @@ class Parallelize:
tables = ' union all '.join(lines)
out = os.path.join(self.athena.output, dest)
sql = load_sql(self.UNION_TABLES, dest = dest,
output = out,
tables = tables)
sql = load_template(self.UNION_TABLES, dest = dest,
output = out,
tables = tables)
return self.athena.execute(sql)