From 3ff2c08c3ae96130cf08c43ae96c881f7bf1f688 Mon Sep 17 00:00:00 2001 From: "Ramiro R. C." Date: Wed, 11 Sep 2024 12:16:03 -0300 Subject: [PATCH] 92 manager sampler evaluator config signature support (#99) * working code * removed config from tokenizer evaluate task * linted --- apps/go/manager/config.sample.json | 8 +- apps/go/manager/records/task.go | 79 ++++---- apps/go/manager/types/config.go | 8 +- .../signatures/model_config_evaluate.py | 184 ++++++++++++++++++ .../signatures/tokenizer_evaluate.py | 48 +---- apps/python/evaluator/worker/main.py | 2 + apps/python/evaluator/workflows/evaluator.py | 8 + .../activities/signatures/config/config.py | 40 ++++ .../activities/signatures/signatures.py | 5 + apps/python/sidecar/main.py | 24 ++- docker-compose/dev/apps/config/manager.json | 12 +- .../morse-poc/apps_configs/manager.json | 12 +- .../temporal/initialize.sh | 18 +- 13 files changed, 341 insertions(+), 107 deletions(-) create mode 100644 apps/python/evaluator/activities/signatures/model_config_evaluate.py create mode 100644 apps/python/sampler/activities/signatures/config/config.py diff --git a/apps/go/manager/config.sample.json b/apps/go/manager/config.sample.json index bc581ee..fd0c36e 100644 --- a/apps/go/manager/config.sample.json +++ b/apps/go/manager/config.sample.json @@ -38,21 +38,21 @@ "frameworks": { "lmeh" : { "task_types": {"any" : "numerical"}, - "task_dependency": {"any" : "signatures:tokenizer:ok"}, + "task_dependency": {"any" : ["signatures:tokenizer:ok", "signatures:config:ok"]}, "schedule_limits": {"any" : "none:none"}, "trigger_minimum": {"any" : "0"} }, "helm" : { "task_types": {"any" : "numerical"}, - "task_dependency": {"any" : "signatures:tokenizer:ok"}, + "task_dependency": {"any" : ["signatures:tokenizer:ok", "signatures:config:ok"]}, "schedule_limits": {"any" : "none:none"}, "trigger_minimum": {"any" : "0"} }, "signatures" : { "task_types": {"any" : "signature"}, - "task_dependency": {"any" : "none:none:none"}, + "task_dependency": {"any" : ["none:none:none"]}, "schedule_limits": {"any" : "1:session"}, - "trigger_minimum": {"tokenizer" : "1"} + "trigger_minimum": {"tokenizer" : "1", "config" : "1"} } } diff --git a/apps/go/manager/records/task.go b/apps/go/manager/records/task.go index 293062f..2ff2c77 100644 --- a/apps/go/manager/records/task.go +++ b/apps/go/manager/records/task.go @@ -181,47 +181,54 @@ func CheckTaskDependency(nodeData *NodeRecord, framework string, task string, co } // Check dependency - frameworkTaskandStatus := strings.Split(taskDep, ":") - if len(frameworkTaskandStatus) != 3 { - l.Error().Str("framework", framework).Str("task", task).Msg("malformed dependency configuration, expected three elements separated by \":\" ") - return false, nil - } - if frameworkTaskandStatus[0] == "none" { - // No dependencies - l.Debug().Str("address", nodeData.Address).Str("service", nodeData.Service).Str("framework", framework).Str("task", task).Msg("No dependency: Dependecy OK") - return true, nil - } - taskType, err := GetTaskType(frameworkTaskandStatus[0], frameworkTaskandStatus[1], configMap, l) - if err != nil { - l.Error().Str("framework", framework).Str("task", task).Str("task type", taskType).Msg("Error getting task type") - return false, err - } - thisTaskRecord, found := GetTaskData(nodeData.ID, taskType, frameworkTaskandStatus[0], frameworkTaskandStatus[1], mongoDB, l) - if !found { - // The task is not even created, we must fail - return false, nil - } else { - // Check the condition - if frameworkTaskandStatus[2] == "present" { - // Task is present, so OK - l.Debug().Str("address", nodeData.Address).Str("service", nodeData.Service).Str("framework", framework).Str("task", task).Msg("Present: Dependecy OK") - return true, nil - } else if frameworkTaskandStatus[2] == "ok" { - // Check for it having a correct value - if thisTaskRecord.IsOK() { - l.Debug().Str("address", nodeData.Address).Str("service", nodeData.Service).Str("framework", framework).Str("task", task).Msg("OK: Dependecy OK") - return true, nil - } + depOK := true + for idxDep := 0; idxDep < len(taskDep); idxDep++ { + // get data from entry + frameworkTaskandStatus := strings.Split(taskDep[idxDep], ":") + if len(frameworkTaskandStatus) != 3 { + l.Error().Str("framework", framework).Str("task", task).Msg("malformed dependency configuration, expected three elements separated by \":\" ") + depOK = false + break + } + if frameworkTaskandStatus[0] == "none" { + // No dependencies + l.Debug().Str("address", nodeData.Address).Str("service", nodeData.Service).Str("framework", framework).Str("task", task).Msg("No dependency: Dependecy OK") + continue + } + taskType, err := GetTaskType(frameworkTaskandStatus[0], frameworkTaskandStatus[1], configMap, l) + if err != nil { + l.Error().Str("framework", framework).Str("task", task).Str("task type", taskType).Msg("Error getting task type") + return false, err + } + thisTaskRecord, found := GetTaskData(nodeData.ID, taskType, frameworkTaskandStatus[0], frameworkTaskandStatus[1], mongoDB, l) + if !found { + // The task is not even created, we must fail + depOK = false + break } else { - l.Error().Str("framework", framework).Str("task", task).Msg("dependency configuration cannot be processed (status type unknown)") - return false, nil + // Check the condition + if frameworkTaskandStatus[2] == "present" { + // Task is present, so OK + l.Debug().Str("address", nodeData.Address).Str("service", nodeData.Service).Str("framework", framework).Str("task", task).Msg("Present: Dependecy OK") + continue + } else if frameworkTaskandStatus[2] == "ok" { + // Check for it having a correct value + if thisTaskRecord.IsOK() { + l.Debug().Str("address", nodeData.Address).Str("service", nodeData.Service).Str("framework", framework).Str("task", task).Msg("OK: Dependecy OK") + continue + } + } else { + l.Error().Str("framework", framework).Str("task", task).Msg("dependency configuration cannot be processed (status type unknown)") + depOK = false + break + } } } - return false, nil + return depOK, nil } -// Analyzes the configuration and checks wheter the triggering the task will +// Analyzes the configuration and checks whether the triggering the task will // break the schedule limits or not (i.e. trigger twice in the same session) func CheckTaskSchedule(taskData TaskInterface, block types.BlockData, configMap map[string]types.FrameworkConfig, l *zerolog.Logger) (bool, error) { @@ -309,7 +316,7 @@ func CheckTaskTriggerMin(taskData TaskInterface, block types.BlockData, configMa // Search for the "any" field taskTriggerMin, ok = frameworkCfg.TriggerMinimum["any"] if !ok { - l.Error().Str("framework", framework).Str("task", task).Msg("cannot find default (or specific) value for task trgger minimum") + l.Error().Str("framework", framework).Str("task", task).Msg("cannot find default (or specific) value for task trigger minimum") err := fmt.Errorf("cannot find default (or specific) value for task trigger minimum") return 0, err } diff --git a/apps/go/manager/types/config.go b/apps/go/manager/types/config.go index 944a60f..8458c00 100644 --- a/apps/go/manager/types/config.go +++ b/apps/go/manager/types/config.go @@ -236,10 +236,10 @@ type Config struct { } type FrameworkConfig struct { - TasksTypes map[string]string `json:"task_types"` - TasksDependency map[string]string `json:"task_dependency"` - ScheduleLimits map[string]string `json:"schedule_limits"` - TriggerMinimum map[string]string `json:"trigger_minimum"` + TasksTypes map[string]string `json:"task_types"` + TasksDependency map[string][]string `json:"task_dependency"` + ScheduleLimits map[string]string `json:"schedule_limits"` + TriggerMinimum map[string]string `json:"trigger_minimum"` } type DevelopConfig struct { diff --git a/apps/python/evaluator/activities/signatures/model_config_evaluate.py b/apps/python/evaluator/activities/signatures/model_config_evaluate.py new file mode 100644 index 0000000..6699d25 --- /dev/null +++ b/apps/python/evaluator/activities/signatures/model_config_evaluate.py @@ -0,0 +1,184 @@ +import json +from datetime import datetime + +from app.app import get_app_config, get_app_logger +from bson import ObjectId +from temporalio import activity +from temporalio.exceptions import ApplicationError + +from packages.python.common.auto_heartbeater import auto_heartbeater +from packages.python.lmeh.utils.mongodb import MongoOperator +from packages.python.lmeh.utils.tokenizers import ( + load_config, + prepare_config, +) +from packages.python.protocol.protocol import ( + PocketNetworkEvaluationTaskRequest, + PocketNetworkMongoDBResultSignature, + PocketNetworkMongoDBConfig, + SignatureSample, + PocketNetworkMongoDBResultBase, +) + + +@activity.defn +@auto_heartbeater +async def model_config_evaluate(args: PocketNetworkEvaluationTaskRequest) -> bool: + """ + Returns a dict where each key is a task name with the evaluation result. + :param args: + :return: + """ + app_config = get_app_config() + eval_logger = get_app_logger("evaluation") + config = app_config["config"] + mongo_client = config["mongo_client"] + mongo_operator = MongoOperator(client=mongo_client) + + try: + try: + task_id_str = args.task_id + args.task_id = ObjectId(args.task_id) + except Exception as e: + raise ApplicationError( + "Bad Task ID format", + str(e), + args.task_id, + type="BadParams", + non_retryable=True, + ) + + # Retrieve all responses + responses = await mongo_operator.retrieve_responses(args.task_id) + if len(responses) != 1: + eval_logger.error(f"Found {len(responses)} responses, only 1 is expected.") + raise ApplicationError( + f"Task ID {args.task_id}: Found {len(responses)} responses, only 1 is expected.", + str(args.task_id), + type="ResponseError", + non_retryable=False, + ) + + # Create the result, empty for now + result = PocketNetworkMongoDBResultSignature( + result_data=PocketNetworkMongoDBResultBase( + task_id=args.task_id, + status=responses[0]["response"]["error_code"], + num_samples=0, + result_height=responses[0]["response"]["height"], + result_time=datetime.today().isoformat(), + ), + signatures=[], + ) + + # Get config jsons + model_config_decoded = False + try: + model_config_jsons = json.loads(responses[0]["response"]["response"]) + model_config_decoded = True + except Exception as e: + eval_logger.debug("Exeption:", Exeption=str(e)) + + model_config_ok = False + if model_config_decoded: + eval_logger.debug( + "Model config found.", model_config_keys=list(model_config_jsons.keys()) + ) + + try: + # Try to load, if this succeds, the model config is OK + temp_path = "/tmp/" + task_id_str + + _config = load_config( + config_objects=model_config_jsons, + wf_id="", + config_ephimeral_path=temp_path, + ) + eval_logger.debug("Config loaded.") + # This creates the structure used in the database, containing the hash + config_jsons_loaded, config_hash_loaded = prepare_config( + _config, CONFIG_EPHIMERAL_PATH=temp_path + ) + # TODO + # For instance, the tokenizer hash is used as the config hash + # in future versions, this should be changed + model_config_mongo_new = PocketNetworkMongoDBConfig( + config=config_jsons_loaded, hash=config_hash_loaded + ) + eval_logger.debug("Config processed.") + + model_config_ok = True + except Exception as e: + # This is not an error is just a failure in retrieval of the model config + eval_logger.info("Cannot load the model config from response.") + eval_logger.debug("Exeption:", Exeption=str(e)) + model_config_ok = False + + model_config_new = False + if model_config_ok: + # check if the model_config exists in db + model_config_db = await mongo_operator.get_config_entry( + model_config_mongo_new.hash + ) + if model_config_db is None: + eval_logger.debug("Model config does not exists.") + # the model config is not tracked, we need to create an entry + model_config_new = True + try: + async with mongo_client.start_transaction() as session: + await mongo_client.db["configs"].insert_many( + [model_config_mongo_new.model_dump(by_alias=True)], + ordered=False, + session=session, + ) + eval_logger.debug("Saved new config to DB.") + except Exception as e: + eval_logger.error("Failed to save model cofig to MongoDB.") + eval_logger.error("Exeption:", Exeption=str(e)) + raise ApplicationError( + "Failed to save model config to MongoDB.", non_retryable=True + ) + + # Update the result with valid data + result.result_data.num_samples = 1 # Always one + result.result_data.status = 0 # OK + result.signatures = [ + SignatureSample( + signature=str(model_config_mongo_new.hash), id=0 + ) # This task has a single sample id + ] + + # Save to results db (a failure is also an answer) + try: + async with mongo_client.start_transaction() as session: + await mongo_client.db["results"].find_one_and_update( + {"result_data.task_id": args.task_id}, + {"$set": result.model_dump(by_alias=True)}, + upsert=True, + session=session, + ) + await mongo_client.db["tasks"].update_one( + {"_id": args.task_id}, + {"$set": {"evaluated": True}}, + session=session, + ) + eval_logger.debug("Saved result to DB.") + except Exception as e: + eval_logger.error("Failed to save Result to MongoDB.") + eval_logger.error("Exception:", Exeption=str(e)) + raise ApplicationError( + "Failed to save result to MongoDB.", non_retryable=True + ) + + eval_logger.info( + "Model Config Status:", + model_config_decoded=model_config_decoded, + model_config_is_valid=model_config_ok, + model_config_is_new=model_config_new, + ) + except Exception as e: + # TODO: enhance drop task logic + await mongo_operator.mark_task_to_drop(args.task_id) + raise e + + return True diff --git a/apps/python/evaluator/activities/signatures/tokenizer_evaluate.py b/apps/python/evaluator/activities/signatures/tokenizer_evaluate.py index 5d0f9df..c95d849 100644 --- a/apps/python/evaluator/activities/signatures/tokenizer_evaluate.py +++ b/apps/python/evaluator/activities/signatures/tokenizer_evaluate.py @@ -11,14 +11,11 @@ from packages.python.lmeh.utils.tokenizers import ( load_tokenizer, prepare_tokenizer, - load_config, - prepare_config, ) from packages.python.protocol.protocol import ( PocketNetworkEvaluationTaskRequest, PocketNetworkMongoDBResultSignature, PocketNetworkMongoDBTokenizer, - PocketNetworkMongoDBConfig, SignatureSample, PocketNetworkMongoDBResultBase, ) @@ -78,9 +75,6 @@ async def tokenizer_evaluate(args: PocketNetworkEvaluationTaskRequest) -> bool: tokenizer_decoded = False try: tokenizer_jsons = json.loads(responses[0]["response"]["response"]) - # extrack config from tokenizer jsons - config_jsons = {"config": tokenizer_jsons.pop("config")} - eval_logger.debug("Config", config_jsons=config_jsons) tokenizer_decoded = True except Exception as e: eval_logger.debug("Exeption:", Exeption=str(e)) @@ -112,26 +106,7 @@ async def tokenizer_evaluate(args: PocketNetworkEvaluationTaskRequest) -> bool: tokenizer=tokenizer_jsons_loaded, hash=tokenizer_hash_loaded ) eval_logger.debug("Tokenizer processed.") - ###################### - ### CONFIG - ##################### - _config = load_config( - config_objects=config_jsons, - wf_id="", - config_ephimeral_path=temp_path, - ) - eval_logger.debug("Config loaded.") - # This creates the structure used in the database, containing the hash - config_jsons_loaded, config_hash_loaded = prepare_config( - _config, CONFIG_EPHIMERAL_PATH=temp_path - ) - # TODO - # For instance, the tokenizer hash is used as the config hash - # in future versions, this should be changed - config_mongo_new = PocketNetworkMongoDBConfig( - config=config_jsons_loaded, hash=tokenizer_hash_loaded - ) - eval_logger.debug("Config processed.") + tokenizer_ok = True except Exception as e: # This is not an error is just a failure in retrieval of tokenizer @@ -172,27 +147,6 @@ async def tokenizer_evaluate(args: PocketNetworkEvaluationTaskRequest) -> bool: signature=str(tokenizer_mongo_new.hash), id=0 ) # This task has a single sample id ] - ###################### - ### CONFIG - ##################### - config_db = await mongo_operator.get_config_entry(config_mongo_new.hash) - if config_db is None: - eval_logger.debug("Config does not exists.") - # the config is not tracked, we need to create an entry - try: - async with mongo_client.start_transaction() as session: - await mongo_client.db["configs"].insert_many( - [config_mongo_new.model_dump(by_alias=True)], - ordered=False, - session=session, - ) - eval_logger.debug("Saved new config to DB.") - except Exception as e: - eval_logger.error("Failed to save Config to MongoDB.") - eval_logger.error("Exeption:", Exeption=str(e)) - raise ApplicationError( - "Failed to save config to MongoDB.", non_retryable=True - ) # Save to results db (a failure is also an answer) try: diff --git a/apps/python/evaluator/worker/main.py b/apps/python/evaluator/worker/main.py index 90f0f11..f5976a7 100644 --- a/apps/python/evaluator/worker/main.py +++ b/apps/python/evaluator/worker/main.py @@ -20,6 +20,7 @@ from activities.get_task_data import get_task_data from activities.lookup_tasks import lookup_tasks from activities.signatures.tokenizer_evaluate import tokenizer_evaluate +from activities.signatures.model_config_evaluate import model_config_evaluate from workflows.evaluator import Evaluator from workflows.lookup_tasks import LookupTasks @@ -104,6 +105,7 @@ async def main(): get_task_data, lmeh_evaluate, tokenizer_evaluate, + model_config_evaluate, ], } diff --git a/apps/python/evaluator/workflows/evaluator.py b/apps/python/evaluator/workflows/evaluator.py index 90f0d43..7aa1c9e 100644 --- a/apps/python/evaluator/workflows/evaluator.py +++ b/apps/python/evaluator/workflows/evaluator.py @@ -7,6 +7,7 @@ from activities.lmeh.evaluate import lmeh_evaluate from activities.get_task_data import get_task_data from activities.signatures.tokenizer_evaluate import tokenizer_evaluate +from activities.signatures.model_config_evaluate import model_config_evaluate from temporalio.common import WorkflowIDReusePolicy from temporalio.workflow import ParentClosePolicy from app.app import get_app_config @@ -44,6 +45,13 @@ async def run(self, args: PocketNetworkEvaluationTaskRequest) -> bool: start_to_close_timeout=timedelta(seconds=300), retry_policy=RetryPolicy(maximum_attempts=2), ) + elif task == "config": + _ = await workflow.execute_activity( + model_config_evaluate, + args, + start_to_close_timeout=timedelta(seconds=300), + retry_policy=RetryPolicy(maximum_attempts=2), + ) else: raise ApplicationError( f"Task {task} of framework {framework} is not implemented yet.", diff --git a/apps/python/sampler/activities/signatures/config/config.py b/apps/python/sampler/activities/signatures/config/config.py new file mode 100644 index 0000000..58f3438 --- /dev/null +++ b/apps/python/sampler/activities/signatures/config/config.py @@ -0,0 +1,40 @@ +from typing import List + +from packages.python.protocol.protocol import ( + PocketNetworkMongoDBInstance, + PocketNetworkMongoDBPrompt, + PocketNetworkMongoDBTask, + RequesterArgs, +) + + +def get_config_task( + args: RequesterArgs, +) -> tuple[ + PocketNetworkMongoDBTask, + List[PocketNetworkMongoDBInstance], + List[PocketNetworkMongoDBPrompt], +]: + # Set call variables + args.method = "GET" + args.path = "/pokt/config" + args.headers = {} + + # Create task + task = PocketNetworkMongoDBTask( + framework="signatures", + requester_args=args, + blacklist=[], + qty=1, # Config is hardcoded to this, no point in asking twice + tasks="config", + total_instances=1, + request_type="", # Remove + ) + # There is a single instance for getting the tokenizer + instance = PocketNetworkMongoDBInstance(task_id=task.id) + # Create the void prompt + prompt = PocketNetworkMongoDBPrompt( + model_config={}, data="", task_id=task.id, instance_id=instance.id, timeout=10 + ) + + return task, [instance], [prompt] diff --git a/apps/python/sampler/activities/signatures/signatures.py b/apps/python/sampler/activities/signatures/signatures.py index 63f40b4..3761610 100644 --- a/apps/python/sampler/activities/signatures/signatures.py +++ b/apps/python/sampler/activities/signatures/signatures.py @@ -8,6 +8,7 @@ # add file path to sys.path sys.path.append(os.path.dirname(os.path.realpath(__file__))) from activities.signatures.tokenizer.tokenizer import get_tokenizer_task +from activities.signatures.config.config import get_config_task from packages.python.common.auto_heartbeater import auto_heartbeater # Custom modules @@ -42,6 +43,10 @@ async def sign_sample(args: PocketNetworkTaskRequest) -> bool: logger.debug("starting tokenizer task sample") task, instances, prompts = get_tokenizer_task(args.requester_args) + elif args.tasks == "config": + logger.debug("starting config task sample") + task, instances, prompts = get_config_task(args.requester_args) + else: logger.error(f"requested task {args.tasks} is not supported") return False diff --git a/apps/python/sidecar/main.py b/apps/python/sidecar/main.py index f2fb04b..9bc90e0 100644 --- a/apps/python/sidecar/main.py +++ b/apps/python/sidecar/main.py @@ -4,7 +4,7 @@ from fastapi.responses import JSONResponse from transformers import AutoConfig, AutoTokenizer -from packages.python.lmeh.utils.tokenizers import _get_config_jsons, prepare_tokenizer +from packages.python.lmeh.utils.tokenizers import prepare_config, prepare_tokenizer ################################################### # SET UP SIDECAR @@ -27,7 +27,9 @@ ) _config = AutoConfig.from_pretrained(config["tokenizer_path"]) -CONFIG_JSON = _get_config_jsons(_config, CONFIG_EPHIMERAL_PATH=CONFIG_EPHIMERAL_PATH) +CONFIG_JSON, CONFIG_HASH = prepare_config( + _config, CONFIG_EPHIMERAL_PATH=CONFIG_EPHIMERAL_PATH +) # add config to tokenizer json TOKENIZER_JSON.update(CONFIG_JSON) @@ -57,3 +59,21 @@ def get_tokenizer(): def get_tokenizer_hash(): logger.debug("returning tokenizer hash") return JSONResponse({"hash": TOKENIZER_HASH}) + + +# ----------------------------------------------- +# Get Full Config +# ----------------------------------------------- +@app.get("/pokt/config") +def get_config(): + logger.debug("returning config data") + return JSONResponse(content=CONFIG_JSON) + + +# ----------------------------------------------- +# Get Config Hash +# ----------------------------------------------- +@app.get("/pokt/config-hash") +def get_config_hash(): + logger.debug("returning config hash") + return JSONResponse({"hash": CONFIG_HASH}) diff --git a/docker-compose/dev/apps/config/manager.json b/docker-compose/dev/apps/config/manager.json index 3ee8630..e6b69e6 100644 --- a/docker-compose/dev/apps/config/manager.json +++ b/docker-compose/dev/apps/config/manager.json @@ -33,15 +33,21 @@ "frameworks": { "lmeh" : { "task_types": {"any" : "numerical"}, - "task_dependency": {"any" : "signatures:tokenizer:ok"} + "task_dependency": {"any" : ["signatures:tokenizer:ok", "signatures:config:ok"]}, + "schedule_limits": {"any" : "none:none"}, + "trigger_minimum": {"any" : "0"} }, "helm" : { "task_types": {"any" : "numerical"}, - "task_dependency": {"any" : "signatures:tokenizer:ok"} + "task_dependency": {"any" : ["signatures:tokenizer:ok", "signatures:config:ok"]}, + "schedule_limits": {"any" : "none:none"}, + "trigger_minimum": {"any" : "0"} }, "signatures" : { "task_types": {"any" : "signature"}, - "task_dependency": {"any" : "none:none:none"} + "task_dependency": {"any" : ["none:none:none"]}, + "schedule_limits": {"any" : "1:session"}, + "trigger_minimum": {"tokenizer" : "1", "config" : "1"} } } } \ No newline at end of file diff --git a/docker-compose/morse-poc/apps_configs/manager.json b/docker-compose/morse-poc/apps_configs/manager.json index 620deb1..de939d8 100644 --- a/docker-compose/morse-poc/apps_configs/manager.json +++ b/docker-compose/morse-poc/apps_configs/manager.json @@ -33,21 +33,15 @@ "frameworks": { "lmeh" : { "task_types": {"any" : "numerical"}, - "task_dependency": {"any" : "signatures:tokenizer:ok"}, - "schedule_limits": {"any" : "none:none"}, - "trigger_minimum": {"any" : "0"} - }, - "helm" : { - "task_types": {"any" : "numerical"}, - "task_dependency": {"any" : "signatures:tokenizer:ok"}, + "task_dependency": {"any" : ["signatures:tokenizer:ok", "signatures:config:ok"]}, "schedule_limits": {"any" : "none:none"}, "trigger_minimum": {"any" : "0"} }, "signatures" : { "task_types": {"any" : "signature"}, - "task_dependency": {"any" : "none:none:none"}, + "task_dependency": {"any" : ["none:none:none"]}, "schedule_limits": {"any" : "1:session"}, - "trigger_minimum": {"tokenizer" : "1"} + "trigger_minimum": {"tokenizer" : "1", "config" : "1"} } } diff --git a/docker-compose/morse-poc/dependencies_configs/temporal/initialize.sh b/docker-compose/morse-poc/dependencies_configs/temporal/initialize.sh index 3f116bb..18e02a8 100755 --- a/docker-compose/morse-poc/dependencies_configs/temporal/initialize.sh +++ b/docker-compose/morse-poc/dependencies_configs/temporal/initialize.sh @@ -16,9 +16,9 @@ keys_services_array=(${keys_services//,/ }) everything="arc_challenge,hellaswag,truthfulqa_mc2,mmlu_abstract_algebra,mmlu_anatomy,mmlu_astronomy,mmlu_business_ethics,mmlu_clinical_knowledge,mmlu_college_biology,mmlu_college_chemistry,mmlu_college_computer_science,mmlu_college_mathematics,mmlu_college_medicine,mmlu_college_physics,mmlu_computer_security,mmlu_conceptual_physics,mmlu_econometrics,mmlu_electrical_engineering,mmlu_elementary_mathematics,mmlu_formal_logic,mmlu_global_facts,mmlu_high_school_biology,mmlu_high_school_chemistry,mmlu_high_school_computer_science,mmlu_high_school_european_history,mmlu_high_school_geography,mmlu_high_school_government_and_politics,mmlu_high_school_macroeconomics,mmlu_high_school_mathematics,mmlu_high_school_microeconomics,mmlu_high_school_physics,mmlu_high_school_psychology,mmlu_high_school_statistics,mmlu_high_school_us_history,mmlu_high_school_world_history,mmlu_human_aging,mmlu_human_sexuality,mmlu_international_law,mmlu_jurisprudence,mmlu_logical_fallacies,mmlu_machine_learning,mmlu_management,mmlu_marketing,mmlu_medical_genetics,mmlu_miscellaneous,mmlu_moral_disputes,mmlu_moral_scenarios,mmlu_nutrition,mmlu_philosophy,mmlu_prehistory,mmlu_professional_accounting,mmlu_professional_law,mmlu_professional_medicine,mmlu_professional_psychology,mmlu_public_relations,mmlu_security_studies,mmlu_sociology,mmlu_us_foreign_policy,mmlu_virology,mmlu_world_religions,winogrande,gsm8k" mmlu="mmlu_abstract_algebra,mmlu_anatomy,mmlu_astronomy,mmlu_business_ethics,mmlu_clinical_knowledge,mmlu_college_biology,mmlu_college_chemistry,mmlu_college_computer_science,mmlu_college_mathematics,mmlu_college_medicine,mmlu_college_physics,mmlu_computer_security,mmlu_conceptual_physics,mmlu_econometrics,mmlu_electrical_engineering,mmlu_elementary_mathematics,mmlu_formal_logic,mmlu_global_facts,mmlu_high_school_biology,mmlu_high_school_chemistry,mmlu_high_school_computer_science,mmlu_high_school_european_history,mmlu_high_school_geography,mmlu_high_school_government_and_politics,mmlu_high_school_macroeconomics,mmlu_high_school_mathematics,mmlu_high_school_microeconomics,mmlu_high_school_physics,mmlu_high_school_psychology,mmlu_high_school_statistics,mmlu_high_school_us_history,mmlu_high_school_world_history,mmlu_human_aging,mmlu_human_sexuality,mmlu_international_law,mmlu_jurisprudence,mmlu_logical_fallacies,mmlu_machine_learning,mmlu_management,mmlu_marketing,mmlu_medical_genetics,mmlu_miscellaneous,mmlu_moral_disputes,mmlu_moral_scenarios,mmlu_nutrition,mmlu_philosophy,mmlu_prehistory,mmlu_professional_accounting,mmlu_professional_law,mmlu_professional_medicine,mmlu_professional_psychology,mmlu_public_relations,mmlu_security_studies,mmlu_sociology,mmlu_us_foreign_policy,mmlu_virology,mmlu_world_religions" heavy="arc_challenge,hellaswag,truthfulqa_mc2,winogrande,gsm8k" -one="gsm8k" +one="mmlu_college_biology" # change this if you want a different set of datasets, by default it create everything -keys=$heavy +keys=$one json_array=$(printf ',"%s"' "${key_array[@]}") json_array="[${json_array:1}]" @@ -72,6 +72,20 @@ for service in "${keys_services_array[@]}"; do --input "{\"service\":\"$service\", \"tests\": [{\"framework\": \"signatures\", \"tasks\": [\"tokenizer\"]}]}" done +for service in "${keys_services_array[@]}"; do + temporal schedule create \ + --schedule-id "lmeh-config-$service" \ + --workflow-id "lmeh-config-$service" \ + --namespace 'pocket-ml-testbench' \ + --workflow-type 'Manager' \ + --task-queue 'manager' \ + --interval '2m' \ + --overlap-policy "Skip" \ + --execution-timeout 120 \ + --task-timeout 120 \ + --input "{\"service\":\"$service\", \"tests\": [{\"framework\": \"signatures\", \"tasks\": [\"config\"]}]}" +done + for service in "${keys_services_array[@]}"; do temporal schedule create \ --schedule-id "f3abbe313689a603a1a6d6a43330d0440a552288-$service" \