ari/minerva
1
0
Fork 0
forked from bellwether/minerva
Code Pull requests Activity

tidying up examples and readme

Browse source
This commit is contained in:
Ari Brown 2024-01-30 18:13:56 -05:00
parent 819bf7abf3
commit e3c11fb1aa
5 changed files with 138 additions and 66 deletions
Download patch file Download diff file Expand all files Collapse all files

30
examples/athena_basic_query.py
View file

@ -3,37 +3,31 @@ import pprint
pp = pprint.PrettyPrinter(indent=4)
# Create the Minerva object which gives you access to the account under the
# profile `hay`
m = minerva.Minerva("hay")
# Get the Athena object
athena = m.athena("s3://haystac-pmo-athena/")
#query = athena.query(
#"""SELECT *
#FROM trajectories.kitware
#WHERE ST_Disjoint(
# ST_GeometryFromText('POLYGON((103.6 1.2151693, 103.6 1.5151693, 104.14797 1.5151693, 104.14797 1.2151693, 103.6 1.2151693))'),
# ST_Point(longitude, latitude)
#)
#""")
# Everything *needs* to have a column in order for parquet to work, so scalar
# values have to be assigned something, so here we use `as count` to create
# a temporary column called `count`
#print(athena.query("select count(*) as count from trajectories.kitware").results().head(1))
# Everything *needs* to have a column in order for unloading to parquet to work,
# so scalar values have to be assigned something, so here we use `as count` to
# create a temporary column called `count`
query = athena.query(
"""
select round(longitude, 3) as lon, count(*) as count
from trajectories.kitware
from trajectories.baseline
where agent = 4
group by round(longitude, 3)
order by count(*) desc
"""
)
data = query.results()
pp.pprint(data.head(10))
# We also get important statistics
print(query.runtime)
#import IPython
#IPython.embed()
print(query.cost)

12
examples/redshift_basic_query.py
View file

@ -3,12 +3,14 @@ import pprint
pp = pprint.PrettyPrinter(indent=4)
m = minerva.Minerva("hay")
red = m.redshift("s3://haystac-pmo-athena/",
db="dev",
cluster="redshift-cluster-1")
query = red.query("select count(*) from myspectrum_schema.kitware where agent = 4")
m = minerva.Minerva("hay-te")
red = m.redshift("s3://haystac-te-athena/",
db = "train",
workgroup = "phase1-trial2")
query = red.query("select agent, st_astext(geom), datetime from public.baseline where agent = 4 limit 200")
data = query.results()
pp.pprint(data.head(10))
print(query.runtime)
print(query.cost)

19
examples/simple_instance.py
View file

@ -1,24 +1,23 @@
from minerva.cluster import Cluster
from minerva.pier import Pier
import minerva
def worker(pier, n=0):
mach = pier.machine(ami = "ami-05a242924e713f80a", # dask on ubuntu 22.04 x86
instance_type = "t3.medium",
username = "ubuntu",
name = f"dask-client-{n}",
name = f"test-{n}",
variables = {"type": "worker",
"number": n})
return mach
pier = Pier("hay",
subnet_id = "subnet-05eb26d8649a093e1", # project-subnet-public1-us-east-1a
sg_groups = ["sg-0f9e555954e863954", # ssh
"sg-0b34a3f7398076545", # default
"sg-04cd2626d91ac093c"], # dask (8786, 8787)
iam = "S3+SSM+CloudWatch+ECR",
key_pair = ("Ari-Brown-HAY", "~/.ssh/Ari-Brown-HAY.pem"))
m = minerva.Minerva("hay")
pier = m.pier(subnet_id = "subnet-05eb26d8649a093e1", # project-subnet-public1-us-east-1a
sg_groups = ["sg-0f9e555954e863954", # ssh
"sg-0b34a3f7398076545"] # default
iam = "S3+SSM+CloudWatch+ECR",
key_pair = ("Ari-Brown-HAY", "~/.ssh/Ari-Brown-HAY.pem"))
mach = worker(pier)
mach.create()
mach.login()
mach.terminate()