ari/minerva
1
0
Fork 0
forked from bellwether/minerva
Code Pull requests Activity

added a missing dependency; added support for redshift queries to return 0 rows

Browse source
This commit is contained in:
Ari Brown 2024-02-01 14:56:48 -05:00
parent 6e249086d2
commit 40d2eefd81
3 changed files with 17 additions and 13 deletions
Download patch file Download diff file Expand all files Collapse all files

22
minerva/redshift.py
View file

@ -137,18 +137,18 @@ drop table temp_data;
status = self.finish()
if status == "FINISHED":
# Track the runtime
self.runtime = self.info_cache['UpdatedAt'] - self.info_cache['CreatedAt']
if self.info_cache['ResultRows'] != 0:
# Because we're using `UNLOAD`, we get a manifest of the files
# that make up our data.
manif = self.out + "manifest"
tmp = self.handler.s3.download(manif)
with open(tmp, "r") as f:
js = json.load(f)
# Because we're using `UNLOAD`, we get a manifest of the files
# that make up our data.
manif = self.out + "manifest"
tmp = self.handler.s3.download(manif)
with open(tmp, "r") as f:
js = json.load(f)
# Filter empty strings
self.files = [e['url'].strip() for e in js['entries'] if e['url'].strip()]
# Filter empty strings
self.files = [e['url'].strip() for e in js['entries'] if e['url'].strip()]
else: # no results returned, so no manifest file was created
self.files = []
return self.files
else: