ari/minerva
1
0
Fork 0
forked from bellwether/minerva
Code Pull requests Activity

improved run_cluster.py; touching up the dask test; upgrade pyarrow dependency for security patch

Browse source
This commit is contained in:
Ari Brown 2023-11-28 11:12:40 -05:00
parent c6280a3826
commit 210e7ebd92
3 changed files with 99 additions and 40 deletions
Download patch file Download diff file Expand all files Collapse all files

25
run_cluster.py
View file

@ -4,9 +4,12 @@ from minerva.pier import Pier
########### PREP ############################
# modin base (built on dask base)
DASK_BASE = "ami-0a8b05658a4f00e52" # dask on ubuntu 22.04 x86
def worker(pier, n):
mach = pier.machine(ami = "ami-0a3df8365ed416816", # dask on ubuntu 22.04 x86
instance_type = "t3.medium",
mach = pier.machine(ami = DASK_BASE,
instance_type = "r5.xlarge",
username = "ubuntu",
name = f"dask-worker-{n}",
variables = {"type": "worker",
@ -15,8 +18,8 @@ def worker(pier, n):
return mach
def scheduler(pier):
mach = pier.machine(ami = "ami-0a3df8365ed416816", # dask on ubuntu 22.04 x86
instance_type = "t3.medium",
mach = pier.machine(ami = DASK_BASE,
instance_type = "r5.xlarge",
username = "ubuntu",
name = f"dask-scheduler",
variables = {"type": "scheduler"},
@ -35,9 +38,17 @@ pier = m.pier(subnet_id = "subnet-05eb26d8649a093e1", # project-subnet-public
cluster = pier.cluster(scheduler, worker, num_workers=int(sys.argv[1]))
cluster.start()
print(cluster.public_location)
print("press enter to terminate cluster")
input()
print()
print(f"dashboard: http://{cluster.scheduler.public_ip}:8787/")
print(f"cluster: {cluster.public_location}")
print()
print("type `exit()` to terminate the cluster")
print()
import IPython
IPython.embed()
cluster.terminate()