ari/minerva
1
0
Fork 0
forked from bellwether/minerva
Code Pull requests Activity

beginning to add web visualization of pools

Browse source
This commit is contained in:
Ari Brown 2024-07-25 09:30:07 -04:00
parent 58f3d7074f
commit 9f8b67b98f
4 changed files with 30 additions and 10 deletions
Download patch file Download diff file Expand all files Collapse all files

11
examples/repartition.py
View file

@ -47,7 +47,7 @@ def repartition(mach, agents):
#####################################
# Prep the work
# Find out how many hours there are in the dataset
pool_size = 1
pool_size = 5
objs = s.m.s3.ls(src_top_level + "year=")
hours = set(["s3://" + '/'.join([o.bucket_name, *o.key.split("/")[0:-1]])
@ -66,7 +66,7 @@ try:
#######################################
# Create the machines
# This also waits for them to be made
pool = m.Pool(s.worker, pool_size)
pool = m.Pool(s.worker, pool_size, web=True)
########################################
# Now that we have the pool, put them to work
@ -74,10 +74,13 @@ try:
# doing that until the list is empty
# First part: sort the individual files
pool.run(sort_hour, data=hours)
#pool.run(sort_hour, data=hours)
# Second part: repartition
pool.run(repartition, data=groups)
#pool.run(repartition, data=groups)
import IPython
IPython.embed()
finally:
pool.terminate()