forked from bellwether/minerva
fixed dataset aggregation; typo; and a bad empty string check
This commit is contained in:
parent
a907be22cc
commit
819bf7abf3
2 changed files with 7 additions and 4 deletions
|
|
@ -10,6 +10,9 @@ with open(file, 'r') as f:
|
|||
txt = f.read()
|
||||
|
||||
for line in txt.split("\n"):
|
||||
if not line.strip():
|
||||
continue
|
||||
|
||||
print(line)
|
||||
athena.cancel(line)
|
||||
|
||||
|
|
|
|||
|
|
@ -282,7 +282,7 @@ class Parallelize:
|
|||
def results(self):
|
||||
self.finish()
|
||||
|
||||
return pa.dataset.dataset([q.results().files for q in self.queries])
|
||||
return pa.dataset.dataset([f for q in self.queries for f in q.results().files])
|
||||
|
||||
|
||||
def union_tables(self, dest):
|
||||
|
|
@ -292,8 +292,8 @@ class Parallelize:
|
|||
tables = ' union all '.join(lines)
|
||||
|
||||
out = os.path.join(self.athena.output, dest)
|
||||
sql = load_sql(self.UNION_TABLES, dest = dest,
|
||||
output = out,
|
||||
tables = tables)
|
||||
sql = load_template(self.UNION_TABLES, dest = dest,
|
||||
output = out,
|
||||
tables = tables)
|
||||
return self.athena.execute(sql)
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue