빈칸모델_runall.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "b4870b9e-470e-4371-8ba2-8eb339070014",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-31T08:12:28.205440Z",
     "iopub.status.busy": "2023-10-31T08:12:28.205293Z",
     "iopub.status.idle": "2023-10-31T08:12:28.208191Z",
     "shell.execute_reply": "2023-10-31T08:12:28.207681Z",
     "shell.execute_reply.started": "2023-10-31T08:12:28.205423Z"
    }
   },
   "outputs": [],
   "source": [
    "from glob import glob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "f65f892b-5920-4259-952f-4096241389c7",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-31T08:12:28.209102Z",
     "iopub.status.busy": "2023-10-31T08:12:28.208832Z",
     "iopub.status.idle": "2023-10-31T08:12:28.215385Z",
     "shell.execute_reply": "2023-10-31T08:12:28.214799Z",
     "shell.execute_reply.started": "2023-10-31T08:12:28.209087Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['./results/all/EleutherAI/polyglot-ko-12.8b',\n",
       " './results/all/HumanF-MarkrAI/pub-llama-13B-v3',\n",
       " './results/all/42MARU/GenAI-llama2-ko-en-platypus',\n",
       " './results/all/kyujinpy/KoR-Orca-Platypus-13B',\n",
       " './results/all/jyoung105/KoR-Orca-Platypus-13B-neft',\n",
       " './results/all/KT-AI/midm-bitext-S-7B-inst-v1',\n",
       " './results/all/hyunseoki/ko-en-llama2-13b']"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ckpts = glob('./results/all/*/*')\n",
    "ckpts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "d6f860f2-2f91-482d-9a83-9b2b9bc6c142",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-31T08:12:28.216270Z",
     "iopub.status.busy": "2023-10-31T08:12:28.216069Z",
     "iopub.status.idle": "2023-10-31T08:12:28.220524Z",
     "shell.execute_reply": "2023-10-31T08:12:28.219943Z",
     "shell.execute_reply.started": "2023-10-31T08:12:28.216255Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['EleutherAI/polyglot-ko-12.8b',\n",
       " 'HumanF-MarkrAI/pub-llama-13B-v3',\n",
       " '42MARU/GenAI-llama2-ko-en-platypus',\n",
       " 'kyujinpy/KoR-Orca-Platypus-13B',\n",
       " 'jyoung105/KoR-Orca-Platypus-13B-neft',\n",
       " 'KT-AI/midm-bitext-S-7B-inst-v1',\n",
       " 'hyunseoki/ko-en-llama2-13b']"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "models = ['/'.join(i.split('/')[-2:]) for i in ckpts]\n",
    "models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "0bc60934-a089-49d0-9873-ac6aaccfee55",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-31T08:12:28.221339Z",
     "iopub.status.busy": "2023-10-31T08:12:28.221185Z",
     "iopub.status.idle": "2023-10-31T08:12:28.224249Z",
     "shell.execute_reply": "2023-10-31T08:12:28.223577Z",
     "shell.execute_reply.started": "2023-10-31T08:12:28.221325Z"
    }
   },
   "outputs": [],
   "source": [
    "from pathlib import Path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "47ac548b-51c6-464f-a15c-96518f7a9b2f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-31T08:12:28.225126Z",
     "iopub.status.busy": "2023-10-31T08:12:28.224969Z",
     "iopub.status.idle": "2023-10-31T08:12:28.228473Z",
     "shell.execute_reply": "2023-10-31T08:12:28.227752Z",
     "shell.execute_reply.started": "2023-10-31T08:12:28.225111Z"
    }
   },
   "outputs": [],
   "source": [
    "BASE_DIR = Path('./results/all')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "70842758-a22b-4144-9bd2-cfa606a47560",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-31T08:12:28.229265Z",
     "iopub.status.busy": "2023-10-31T08:12:28.229111Z",
     "iopub.status.idle": "2023-10-31T08:12:28.473550Z",
     "shell.execute_reply": "2023-10-31T08:12:28.472165Z",
     "shell.execute_reply.started": "2023-10-31T08:12:28.229250Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "python not found\r\n"
     ]
    }
   ],
   "source": [
    "!which python"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "1fd4d5e4-8926-443c-84fb-575d3c3b68df",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-31T08:14:36.269153Z",
     "iopub.status.busy": "2023-10-31T08:14:36.268321Z",
     "iopub.status.idle": "2023-10-31T08:14:36.282640Z",
     "shell.execute_reply": "2023-10-31T08:14:36.281300Z",
     "shell.execute_reply.started": "2023-10-31T08:14:36.269106Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CUDA_VISIBLE_DEVICES=\"###\" \\\n",
      "python main.py --model hf-causal-experimental --model_args pretrained=EleutherAI/polyglot-ko-12.8b,use_accelerate=true,trust_remote_code=true --tasks kobest_hellaswag,kobest_copa,kobest_boolq,kobest_sentineg,kohatespeech,kohatespeech_apeach,kohatespeech_gen_bias,korunsmile,nsmc,pawsx_ko --num_fewshot 50 --no_cache --batch_size 4 --output_path results/all/EleutherAI/polyglot-ko-12.8b/50_shot.json\n",
      "\n",
      "CUDA_VISIBLE_DEVICES=\"###\" \\\n",
      "python main.py --model hf-causal-experimental --model_args pretrained=42MARU/GenAI-llama2-ko-en-platypus,use_accelerate=true,trust_remote_code=true --tasks kobest_hellaswag,kobest_copa,kobest_boolq,kobest_sentineg,kohatespeech,kohatespeech_apeach,kohatespeech_gen_bias,korunsmile,nsmc,pawsx_ko --num_fewshot 50 --no_cache --batch_size 4 --output_path results/all/42MARU/GenAI-llama2-ko-en-platypus/50_shot.json\n",
      "\n",
      "CUDA_VISIBLE_DEVICES=\"###\" \\\n",
      "python main.py --model hf-causal-experimental --model_args pretrained=kyujinpy/KoR-Orca-Platypus-13B,use_accelerate=true,trust_remote_code=true --tasks kobest_hellaswag,kobest_copa,kobest_boolq,kobest_sentineg,kohatespeech,kohatespeech_apeach,kohatespeech_gen_bias,korunsmile,nsmc,pawsx_ko --num_fewshot 50 --no_cache --batch_size 4 --output_path results/all/kyujinpy/KoR-Orca-Platypus-13B/50_shot.json\n",
      "\n",
      "CUDA_VISIBLE_DEVICES=\"###\" \\\n",
      "python main.py --model hf-causal-experimental --model_args pretrained=KT-AI/midm-bitext-S-7B-inst-v1,use_accelerate=true,trust_remote_code=true --tasks kobest_hellaswag,kobest_copa,kobest_boolq,kobest_sentineg,kohatespeech,kohatespeech_apeach,kohatespeech_gen_bias,korunsmile,nsmc,pawsx_ko --num_fewshot 50 --no_cache --batch_size 4 --output_path results/all/KT-AI/midm-bitext-S-7B-inst-v1/50_shot.json\n",
      "\n",
      "CUDA_VISIBLE_DEVICES=\"###\" \\\n",
      "python main.py --model hf-causal-experimental --model_args pretrained=hyunseoki/ko-en-llama2-13b,use_accelerate=true,trust_remote_code=true --tasks kobest_hellaswag,kobest_copa,kobest_boolq,kobest_sentineg,kohatespeech,kohatespeech_apeach,kohatespeech_gen_bias,korunsmile,nsmc,pawsx_ko --num_fewshot 5 --no_cache --batch_size 4 --output_path results/all/hyunseoki/ko-en-llama2-13b/5_shot.json\n",
      "\n",
      "CUDA_VISIBLE_DEVICES=\"###\" \\\n",
      "python main.py --model hf-causal-experimental --model_args pretrained=hyunseoki/ko-en-llama2-13b,use_accelerate=true,trust_remote_code=true --tasks kobest_hellaswag,kobest_copa,kobest_boolq,kobest_sentineg,kohatespeech,kohatespeech_apeach,kohatespeech_gen_bias,korunsmile,nsmc,pawsx_ko --num_fewshot 10 --no_cache --batch_size 4 --output_path results/all/hyunseoki/ko-en-llama2-13b/10_shot.json\n",
      "\n",
      "CUDA_VISIBLE_DEVICES=\"###\" \\\n",
      "python main.py --model hf-causal-experimental --model_args pretrained=hyunseoki/ko-en-llama2-13b,use_accelerate=true,trust_remote_code=true --tasks kobest_hellaswag,kobest_copa,kobest_boolq,kobest_sentineg,kohatespeech,kohatespeech_apeach,kohatespeech_gen_bias,korunsmile,nsmc,pawsx_ko --num_fewshot 50 --no_cache --batch_size 4 --output_path results/all/hyunseoki/ko-en-llama2-13b/50_shot.json\n",
      "\n"
     ]
    }
   ],
   "source": [
    "RESULT_DIR='results/all'\n",
    "TASKS='kobest_hellaswag,kobest_copa,kobest_boolq,kobest_sentineg,kohatespeech,kohatespeech_apeach,kohatespeech_gen_bias,korunsmile,nsmc,pawsx_ko'\n",
    "for model in models:\n",
    "    for n in [0, 5, 10, 50]:\n",
    "        if (BASE_DIR / model / f'{n}_shot.json').exists():\n",
    "            continue\n",
    "        else:\n",
    "            print(f'''CUDA_VISIBLE_DEVICES=\"###\" \\\\\n",
    "python main.py \\\n",
    "--model hf-causal-experimental \\\n",
    "--model_args pretrained={model},use_accelerate=true,trust_remote_code=true \\\n",
    "--tasks {TASKS} \\\n",
    "--num_fewshot {n} \\\n",
    "--no_cache \\\n",
    "--batch_size 4 \\\n",
    "--output_path {RESULT_DIR}/{model}/{n}_shot.json''')\n",
    "            print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bd4ffe67-de0c-4d46-bd1b-7a5be84e99ab",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-31T08:12:28.496504Z",
     "iopub.status.busy": "2023-10-31T08:12:28.495862Z",
     "iopub.status.idle": "2023-10-31T08:12:28.502457Z",
     "shell.execute_reply": "2023-10-31T08:12:28.501141Z",
     "shell.execute_reply.started": "2023-10-31T08:12:28.496450Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}