ari/minerva
1
0
Fork 0
forked from bellwether/minerva
Code Pull requests Activity

better redshift support, working on dask stuff

Browse source
This commit is contained in:
Ari Brown 2023-12-21 14:23:58 -05:00
parent e854a93e60
commit 5eb8471081
5 changed files with 86 additions and 175 deletions
Download patch file Download diff file Expand all files Collapse all files

12
cluster/run_cluster.py
View file

@ -5,10 +5,16 @@ from minerva.pier import Pier
########### PREP ############################
DASK_BASE = "ami-0399a4f70ca684620" # dask on ubuntu 22.04 x86
NUM_WORK = int(sys.argv[1])
if sys.argv[2] == "large":
WORKER_TYPE = "r5.xlarge"
else:
WORKER_TYPE = "m5.large"
def worker(pier, n):
mach = pier.machine(ami = DASK_BASE,
instance_type = "m5.large",
instance_type = WORKER_TYPE,
username = "ubuntu",
name = f"dask-worker-{n}",
variables = {"type": "worker",
@ -18,7 +24,7 @@ def worker(pier, n):
def scheduler(pier):
mach = pier.machine(ami = DASK_BASE,
instance_type = "m5.large", # "r5.xlarge",
instance_type = "m5.large",
username = "ubuntu",
name = f"dask-scheduler",
variables = {"type": "scheduler"},
@ -35,7 +41,7 @@ pier = m.pier(subnet_id = "subnet-05eb26d8649a093e1", # project-subnet-public
key_pair = ("Ari-Brown-HAY", "~/.ssh/Ari-Brown-HAY.pem"),
iam = "Minerva")
cluster = pier.cluster(scheduler, worker, num_workers=int(sys.argv[1]))
cluster = pier.cluster(scheduler, worker, num_workers=NUM_WORK)
cluster.start()
print()