forked from bellwether/minerva
added a missing dependency; added support for redshift queries to return 0 rows
This commit is contained in:
parent
6e249086d2
commit
40d2eefd81
3 changed files with 17 additions and 13 deletions
|
|
@ -9,7 +9,10 @@ red = m.redshift("s3://haystac-te-athena/",
|
|||
workgroup = "phase1-trial2")
|
||||
|
||||
query = red.query("""select agent, st_astext(geom), timestamp from
|
||||
public.baseline_v1 where agent = 44 limit 200""")
|
||||
public.baseline_v1 where agent = 4 limit 200""")
|
||||
|
||||
import IPython
|
||||
IPython.embed()
|
||||
|
||||
data = query.results()
|
||||
|
||||
|
|
|
|||
|
|
@ -137,18 +137,18 @@ drop table temp_data;
|
|||
status = self.finish()
|
||||
|
||||
if status == "FINISHED":
|
||||
# Track the runtime
|
||||
self.runtime = self.info_cache['UpdatedAt'] - self.info_cache['CreatedAt']
|
||||
if self.info_cache['ResultRows'] != 0:
|
||||
# Because we're using `UNLOAD`, we get a manifest of the files
|
||||
# that make up our data.
|
||||
manif = self.out + "manifest"
|
||||
tmp = self.handler.s3.download(manif)
|
||||
with open(tmp, "r") as f:
|
||||
js = json.load(f)
|
||||
|
||||
# Because we're using `UNLOAD`, we get a manifest of the files
|
||||
# that make up our data.
|
||||
manif = self.out + "manifest"
|
||||
tmp = self.handler.s3.download(manif)
|
||||
with open(tmp, "r") as f:
|
||||
js = json.load(f)
|
||||
|
||||
# Filter empty strings
|
||||
self.files = [e['url'].strip() for e in js['entries'] if e['url'].strip()]
|
||||
# Filter empty strings
|
||||
self.files = [e['url'].strip() for e in js['entries'] if e['url'].strip()]
|
||||
else: # no results returned, so no manifest file was created
|
||||
self.files = []
|
||||
|
||||
return self.files
|
||||
else:
|
||||
|
|
|
|||
|
|
@ -23,4 +23,5 @@ joblib = "^1.1.0"
|
|||
fabric = "^3.0.0"
|
||||
s3fs = "2023.6.0"
|
||||
mako = ">1.2.0"
|
||||
dask[distributed] = ">2023.11.0"
|
||||
dask = ">2023.11.0"
|
||||
distributed = ">2023.11.0"
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue