From 9b4de7407d80a8e3716d5fb3f48005499a666c72 Mon Sep 17 00:00:00 2001 From: Erlich Sefi Date: Mon, 21 Aug 2023 07:56:04 +0000 Subject: [PATCH 01/47] docker --- .devcontainer/devcontainer.json | 20 ++++++++++++++++++++ Dockerfile | 28 ++++++++++++++++++++++++++++ requirements.txt | 1 + 3 files changed, 49 insertions(+) create mode 100644 .devcontainer/devcontainer.json create mode 100644 Dockerfile create mode 100644 requirements.txt diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000..df73383 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,20 @@ +{ + "build": { + "dockerfile": "../Dockerfile", + "target":"test", + "args": { + "PY_VERSION":"3.11.0" + } + }, + "customizations": { + "vscode": { + "extensions": [ + "ms-python.python", + "ms-python.vscode-pylance", + "ms-toolsai.jupyter" + ] + } + }, + + "forwardPorts": [3000] + } \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..044b082 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,28 @@ +#syntax=docker/dockerfile:1 + +FROM debian:latest as base +WORKDIR /usr/src/app +ARG PY_VERSION="3.11.0" + +# setting the enviroment +RUN apt-get update -y && \ + apt-get install python3-pip -y && \ + apt-get install dieharder -y && \ + apt-get install wget -y && \ + apt-get clean && \ + apt-get autoremove + +# setup python +ENV HOME="/root" +WORKDIR ${HOME} +RUN apt-get install -y git libbz2-dev libncurses-dev libreadline-dev libffi-dev libssl-dev +RUN git clone --depth=1 https://github.com/pyenv/pyenv.git .pyenv +ENV PYENV_ROOT="${HOME}/.pyenv" +ENV PATH="${PYENV_ROOT}/shims:${PYENV_ROOT}/bin:${PATH}" + +RUN pyenv install $PY_VERSION +RUN pyenv global $PY_VERSION + +# setup code +COPY . . +RUN pip install -r requirements.txt \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..2067dce --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +poetry==1.4.2 \ No newline at end of file From 28eee2464748bdf9ead67dae5e9be57ba70b4fb8 Mon Sep 17 00:00:00 2001 From: Erlich Sefi Date: Mon, 21 Aug 2023 14:47:31 +0300 Subject: [PATCH 02/47] Update devcontainer.json --- .devcontainer/devcontainer.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index df73383..765d63d 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,7 +1,6 @@ { "build": { "dockerfile": "../Dockerfile", - "target":"test", "args": { "PY_VERSION":"3.11.0" } @@ -17,4 +16,4 @@ }, "forwardPorts": [3000] - } \ No newline at end of file + } From fa55d6f4bafccd94ee1f09616fb7f667aa0c3855 Mon Sep 17 00:00:00 2001 From: Erlich Sefi Date: Mon, 21 Aug 2023 14:59:20 +0000 Subject: [PATCH 03/47] remove requirements --- Dockerfile | 3 ++- requirements.txt | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) delete mode 100644 requirements.txt diff --git a/Dockerfile b/Dockerfile index 044b082..221b9cc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,4 +25,5 @@ RUN pyenv global $PY_VERSION # setup code COPY . . -RUN pip install -r requirements.txt \ No newline at end of file +RUN pip install poetry==1.4.2 +RUN poetry install \ No newline at end of file diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 2067dce..0000000 --- a/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -poetry==1.4.2 \ No newline at end of file From 1a9a7eca53c1e7aa48e92ad1c6e8635647f8efd5 Mon Sep 17 00:00:00 2001 From: Erlich Sefi Date: Mon, 21 Aug 2023 15:16:08 +0000 Subject: [PATCH 04/47] docker --- Dockerfile | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 221b9cc..7b41f16 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,4 +26,9 @@ RUN pyenv global $PY_VERSION # setup code COPY . . RUN pip install poetry==1.4.2 -RUN poetry install \ No newline at end of file +RUN poetry install + + +# run test +FROM base as test +RUN make test \ No newline at end of file From 707f0684660139c55cad2ba281f70d036f2cd87a Mon Sep 17 00:00:00 2001 From: Erlich Sefi Date: Mon, 21 Aug 2023 18:49:52 +0000 Subject: [PATCH 05/47] first docker --- .github/workflows/tests.yml | 32 +++++++++++++------------------- Dockerfile | 32 +++++++------------------------- 2 files changed, 20 insertions(+), 44 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index cc077d5..f6a7c48 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -5,9 +5,6 @@ on: branches: [main] pull_request: -env: - POETRY_VERSION: "1.4.2" - jobs: build: runs-on: ubuntu-latest @@ -15,19 +12,16 @@ jobs: matrix: python-version: - "3.10" - steps: - - uses: actions/checkout@v3 - - name: Install poetry - run: | - pipx install poetry==$POETRY_VERSION - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - cache: poetry - - name: Install dependencies - run: | - poetry install - - name: Run all tests - run: | - make tests + - name: Checkout + uses: actions/checkout@v3 + - name: Free disk space + run: | + df --human-readable + sudo apt clean + docker 2>/dev/null 1>&2 rmi $(docker image ls --all --quiet) || true + rm --recursive --force "$AGENT_TOOLSDIRECTORY" + df --human-readable + - name: Build with Docker + run: docker build -t chromegpt:test --target test . + - name: Test with pytest + run: docker run --rm --name test-run chromegpt:test diff --git a/Dockerfile b/Dockerfile index 7b41f16..b23b083 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,34 +1,16 @@ -#syntax=docker/dockerfile:1 +FROM python:3.8 as base -FROM debian:latest as base -WORKDIR /usr/src/app -ARG PY_VERSION="3.11.0" - -# setting the enviroment -RUN apt-get update -y && \ - apt-get install python3-pip -y && \ - apt-get install dieharder -y && \ - apt-get install wget -y && \ - apt-get clean && \ - apt-get autoremove - -# setup python -ENV HOME="/root" -WORKDIR ${HOME} -RUN apt-get install -y git libbz2-dev libncurses-dev libreadline-dev libffi-dev libssl-dev -RUN git clone --depth=1 https://github.com/pyenv/pyenv.git .pyenv -ENV PYENV_ROOT="${HOME}/.pyenv" -ENV PATH="${PYENV_ROOT}/shims:${PYENV_ROOT}/bin:${PATH}" - -RUN pyenv install $PY_VERSION -RUN pyenv global $PY_VERSION +WORKDIR /app # setup code COPY . . RUN pip install poetry==1.4.2 -RUN poetry install + +# Install dependencies using Poetry +RUN poetry config virtualenvs.create false && \ + poetry install --no-interaction --no-ansi # run test FROM base as test -RUN make test \ No newline at end of file +CMD make tests \ No newline at end of file From 94e65145ec276370aa9f2b2aea9bdde55d5798e6 Mon Sep 17 00:00:00 2001 From: Erlich Sefi Date: Mon, 21 Aug 2023 19:05:20 +0000 Subject: [PATCH 06/47] docker testing --- .github/workflows/lint.yml | 30 +++++++++++++----------------- .github/workflows/tests.yml | 27 ++++++++++++++------------- Dockerfile | 4 +++- 3 files changed, 30 insertions(+), 31 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index c91fdb9..e9f4c85 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -5,9 +5,6 @@ on: branches: [main] pull_request: -env: - POETRY_VERSION: "1.4.2" - jobs: build: runs-on: ubuntu-latest @@ -16,18 +13,17 @@ jobs: python-version: - "3.10" steps: - - uses: actions/checkout@v3 - - name: Install poetry - run: | - pipx install poetry==$POETRY_VERSION - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - cache: poetry - - name: Install dependencies + - name: Checkout + uses: actions/checkout@v3 + - name: Free disk space run: | - poetry install - - name: Analysing the code with our lint - run: | - make lint + df --human-readable + sudo apt clean + docker 2>/dev/null 1>&2 rmi $(docker image ls --all --quiet) || true + rm --recursive --force "$AGENT_TOOLSDIRECTORY" + df --human-readable + - name: Build with Docker + run: docker build -t chromegpt --target test --build-arg MAKE=lint . + - name: Test with pytest + run: docker run --rm --name test-run chromegpt:test + diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f6a7c48..fca09fb 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -12,16 +12,17 @@ jobs: matrix: python-version: - "3.10" - - name: Checkout - uses: actions/checkout@v3 - - name: Free disk space - run: | - df --human-readable - sudo apt clean - docker 2>/dev/null 1>&2 rmi $(docker image ls --all --quiet) || true - rm --recursive --force "$AGENT_TOOLSDIRECTORY" - df --human-readable - - name: Build with Docker - run: docker build -t chromegpt:test --target test . - - name: Test with pytest - run: docker run --rm --name test-run chromegpt:test + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Free disk space + run: | + df --human-readable + sudo apt clean + docker 2>/dev/null 1>&2 rmi $(docker image ls --all --quiet) || true + rm --recursive --force "$AGENT_TOOLSDIRECTORY" + df --human-readable + - name: Build with Docker + run: docker build -t chromegpt --target test --build-arg MAKE=test . + - name: Test with pytest + run: docker run --rm --name test-run chromegpt:test diff --git a/Dockerfile b/Dockerfile index b23b083..877623f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,4 +13,6 @@ RUN poetry config virtualenvs.create false && \ # run test FROM base as test -CMD make tests \ No newline at end of file +# Set the environment variable +ENV MAKE="test" +CMD make $MAKE \ No newline at end of file From e970547bf811b5f88e94ca3c7bd7b6fc8fbbf4d9 Mon Sep 17 00:00:00 2001 From: Erlich Sefi Date: Mon, 21 Aug 2023 19:09:38 +0000 Subject: [PATCH 07/47] . --- .github/workflows/lint.yml | 3 +-- .github/workflows/tests.yml | 4 ++-- Dockerfile | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index e9f4c85..416c3f6 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -25,5 +25,4 @@ jobs: - name: Build with Docker run: docker build -t chromegpt --target test --build-arg MAKE=lint . - name: Test with pytest - run: docker run --rm --name test-run chromegpt:test - + run: docker run --rm --name test-run chromegpt diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index fca09fb..2752440 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -23,6 +23,6 @@ jobs: rm --recursive --force "$AGENT_TOOLSDIRECTORY" df --human-readable - name: Build with Docker - run: docker build -t chromegpt --target test --build-arg MAKE=test . + run: docker build -t chromegpt --target test --build-arg MAKE=tests . - name: Test with pytest - run: docker run --rm --name test-run chromegpt:test + run: docker run --rm --name test-run chromegpt diff --git a/Dockerfile b/Dockerfile index 877623f..1207664 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,5 +14,5 @@ RUN poetry config virtualenvs.create false && \ # run test FROM base as test # Set the environment variable -ENV MAKE="test" +ENV MAKE="tests" CMD make $MAKE \ No newline at end of file From 3102182e469bba28cef0ae9ce122894f5ffafe48 Mon Sep 17 00:00:00 2001 From: Erlich Sefi Date: Mon, 21 Aug 2023 19:16:25 +0000 Subject: [PATCH 08/47] . --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 1207664..d4eae8a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,5 +14,5 @@ RUN poetry config virtualenvs.create false && \ # run test FROM base as test # Set the environment variable -ENV MAKE="tests" +ARG MAKE="tests" CMD make $MAKE \ No newline at end of file From 7cf302f8319ae1e58f74c48b555789b2a423cd33 Mon Sep 17 00:00:00 2001 From: Erlich Sefi Date: Mon, 21 Aug 2023 19:36:35 +0000 Subject: [PATCH 09/47] seleuim docker --- chromegpt/tools/selenium.py | 8 ++++++-- docker-compose.yaml | 16 ++++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) create mode 100644 docker-compose.yaml diff --git a/chromegpt/tools/selenium.py b/chromegpt/tools/selenium.py index 2755b40..8d4c954 100644 --- a/chromegpt/tools/selenium.py +++ b/chromegpt/tools/selenium.py @@ -37,14 +37,18 @@ class SeleniumWrapper: selenium = SeleniumWrapper() """ - def __init__(self, headless: bool = False) -> None: + def __init__(self, headless: bool = False, docker:bool = True) -> None: """Initialize Selenium and start interactive session.""" chrome_options = Options() + if headless: chrome_options.add_argument("--headless") else: chrome_options.add_argument("--start-maximized") - self.driver = webdriver.Chrome(options=chrome_options) + if docker: + self.driver = webdriver.Remote("http://host.docker.internal:4444/wd/hub",options=chrome_options) + else: + self.driver = webdriver.Chrome(options=chrome_options) self.driver.implicitly_wait(5) # Wait 5 seconds for elements to load def __del__(self) -> None: diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 0000000..47563dd --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,16 @@ +version: '3' +services: + chromegpt: + build: + context: . + dockerfile: Dockerfile + target: test + args: + - MAKE=tests + + selenium-chrome: + image: selenium/standalone-chrome:latest + ports: + - "4444:4444" + - "7900:7900" + shm_size: "2g" \ No newline at end of file From c977c35ade85bfc58c325df039f5171ab36636dc Mon Sep 17 00:00:00 2001 From: Erlich Sefi Date: Mon, 21 Aug 2023 20:14:31 +0000 Subject: [PATCH 10/47] . --- chromegpt/tools/utils.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/chromegpt/tools/utils.py b/chromegpt/tools/utils.py index d42c3e9..c078f64 100644 --- a/chromegpt/tools/utils.py +++ b/chromegpt/tools/utils.py @@ -4,6 +4,7 @@ from typing import List, Optional from selenium.webdriver.chrome.webdriver import WebDriver +from selenium.webdriver.remote.wendriver import WebDriver as RemoteWebDriver from selenium.webdriver.common.by import By from selenium.webdriver.remote.webelement import WebElement from unidecode import unidecode @@ -14,7 +15,7 @@ def is_complete_sentence(text: str) -> bool: return re.search(r"[.!?]\s*$", text) is not None -def get_all_text_elements(driver: WebDriver) -> List[str]: +def get_all_text_elements(driver: [WebDriver,RemoteWebDriver]) -> List[str]: xpath = ( "//*[not(self::script or self::style or" " self::noscript)][string-length(normalize-space(text())) > 0]" @@ -30,7 +31,7 @@ def get_all_text_elements(driver: WebDriver) -> List[str]: return texts -def find_interactable_elements(driver: WebDriver) -> List[str]: +def find_interactable_elements(driver: [WebDriver,RemoteWebDriver]) -> List[str]: """Find all interactable elements on the page.""" # Extract interactable components (buttons and links) buttons = driver.find_elements(By.XPATH, "//button") @@ -62,7 +63,7 @@ def prettify_text(text: str, limit: Optional[int] = None) -> str: return text -def element_completely_viewable(driver: WebDriver, elem: WebElement) -> bool: +def element_completely_viewable(driver: [WebDriver,RemoteWebDriver], elem: WebElement) -> bool: """Check if an element is completely viewable in the browser window.""" elem_left_bound = elem.location.get("x") elem_top_bound = elem.location.get("y") From eca0fc3cddb034a46f7b5e3e63bc41643336461c Mon Sep 17 00:00:00 2001 From: Erlich Sefi Date: Mon, 21 Aug 2023 20:15:40 +0000 Subject: [PATCH 11/47] . --- chromegpt/tools/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chromegpt/tools/utils.py b/chromegpt/tools/utils.py index c078f64..7f17c8d 100644 --- a/chromegpt/tools/utils.py +++ b/chromegpt/tools/utils.py @@ -4,7 +4,7 @@ from typing import List, Optional from selenium.webdriver.chrome.webdriver import WebDriver -from selenium.webdriver.remote.wendriver import WebDriver as RemoteWebDriver +from selenium.webdriver.remote.webdriver import WebDriver as RemoteWebDriver from selenium.webdriver.common.by import By from selenium.webdriver.remote.webelement import WebElement from unidecode import unidecode From e6f3a8bf29519dc063b80f73cd53c1d54551ab60 Mon Sep 17 00:00:00 2001 From: Erlich Sefi Date: Mon, 21 Aug 2023 20:25:48 +0000 Subject: [PATCH 12/47] . --- chromegpt/tools/utils.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/chromegpt/tools/utils.py b/chromegpt/tools/utils.py index 7f17c8d..bb8ac86 100644 --- a/chromegpt/tools/utils.py +++ b/chromegpt/tools/utils.py @@ -1,7 +1,7 @@ """Utils for chromegpt tools.""" import re -from typing import List, Optional +from typing import List, Optional, Union from selenium.webdriver.chrome.webdriver import WebDriver from selenium.webdriver.remote.webdriver import WebDriver as RemoteWebDriver @@ -15,7 +15,7 @@ def is_complete_sentence(text: str) -> bool: return re.search(r"[.!?]\s*$", text) is not None -def get_all_text_elements(driver: [WebDriver,RemoteWebDriver]) -> List[str]: +def get_all_text_elements(driver: Union[WebDriver,RemoteWebDriver]) -> List[str]: xpath = ( "//*[not(self::script or self::style or" " self::noscript)][string-length(normalize-space(text())) > 0]" @@ -31,7 +31,7 @@ def get_all_text_elements(driver: [WebDriver,RemoteWebDriver]) -> List[str]: return texts -def find_interactable_elements(driver: [WebDriver,RemoteWebDriver]) -> List[str]: +def find_interactable_elements(driver: Union[WebDriver,RemoteWebDriver]) -> List[str]: """Find all interactable elements on the page.""" # Extract interactable components (buttons and links) buttons = driver.find_elements(By.XPATH, "//button") @@ -63,7 +63,7 @@ def prettify_text(text: str, limit: Optional[int] = None) -> str: return text -def element_completely_viewable(driver: [WebDriver,RemoteWebDriver], elem: WebElement) -> bool: +def element_completely_viewable(driver: Union[WebDriver,RemoteWebDriver], elem: WebElement) -> bool: """Check if an element is completely viewable in the browser window.""" elem_left_bound = elem.location.get("x") elem_top_bound = elem.location.get("y") From f3ba82de60d229b7d3efcbe8b8ce200011266aa6 Mon Sep 17 00:00:00 2001 From: Erlich Sefi Date: Mon, 21 Aug 2023 20:29:01 +0000 Subject: [PATCH 13/47] black --- chromegpt/tools/selenium.py | 11 ++++++----- chromegpt/tools/utils.py | 8 +++++--- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/chromegpt/tools/selenium.py b/chromegpt/tools/selenium.py index 8d4c954..627df2a 100644 --- a/chromegpt/tools/selenium.py +++ b/chromegpt/tools/selenium.py @@ -37,16 +37,18 @@ class SeleniumWrapper: selenium = SeleniumWrapper() """ - def __init__(self, headless: bool = False, docker:bool = True) -> None: + def __init__(self, headless: bool = False, docker: bool = True) -> None: """Initialize Selenium and start interactive session.""" chrome_options = Options() - + if headless: chrome_options.add_argument("--headless") else: chrome_options.add_argument("--start-maximized") if docker: - self.driver = webdriver.Remote("http://host.docker.internal:4444/wd/hub",options=chrome_options) + self.driver = webdriver.Remote( + "http://host.docker.internal:4444/wd/hub", options=chrome_options + ) else: self.driver = webdriver.Chrome(options=chrome_options) self.driver.implicitly_wait(5) # Wait 5 seconds for elements to load @@ -74,8 +76,7 @@ def google_search(self, query: str) -> str: results = self._get_google_search_results() return ( "Which url would you like to goto? Provide the full url starting with http" - " or https to goto: " - + json.dumps(results) + " or https to goto: " + json.dumps(results) ) def _get_google_search_results(self) -> List[Dict[str, Any]]: diff --git a/chromegpt/tools/utils.py b/chromegpt/tools/utils.py index bb8ac86..d42b784 100644 --- a/chromegpt/tools/utils.py +++ b/chromegpt/tools/utils.py @@ -15,7 +15,7 @@ def is_complete_sentence(text: str) -> bool: return re.search(r"[.!?]\s*$", text) is not None -def get_all_text_elements(driver: Union[WebDriver,RemoteWebDriver]) -> List[str]: +def get_all_text_elements(driver: Union[WebDriver, RemoteWebDriver]) -> List[str]: xpath = ( "//*[not(self::script or self::style or" " self::noscript)][string-length(normalize-space(text())) > 0]" @@ -31,7 +31,7 @@ def get_all_text_elements(driver: Union[WebDriver,RemoteWebDriver]) -> List[str] return texts -def find_interactable_elements(driver: Union[WebDriver,RemoteWebDriver]) -> List[str]: +def find_interactable_elements(driver: Union[WebDriver, RemoteWebDriver]) -> List[str]: """Find all interactable elements on the page.""" # Extract interactable components (buttons and links) buttons = driver.find_elements(By.XPATH, "//button") @@ -63,7 +63,9 @@ def prettify_text(text: str, limit: Optional[int] = None) -> str: return text -def element_completely_viewable(driver: Union[WebDriver,RemoteWebDriver], elem: WebElement) -> bool: +def element_completely_viewable( + driver: Union[WebDriver, RemoteWebDriver], elem: WebElement +) -> bool: """Check if an element is completely viewable in the browser window.""" elem_left_bound = elem.location.get("x") elem_top_bound = elem.location.get("y") From 5d36bd22b8a25072f784da10c3602aeb102629b9 Mon Sep 17 00:00:00 2001 From: Erlich Sefi Date: Mon, 21 Aug 2023 20:53:15 +0000 Subject: [PATCH 14/47] . --- .devcontainer/devcontainer.json | 29 +++++++++++++---------------- docker-compose.yaml | 8 ++++++-- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 765d63d..e73c4e0 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,19 +1,16 @@ { - "build": { - "dockerfile": "../Dockerfile", - "args": { - "PY_VERSION":"3.11.0" - } - }, - "customizations": { + "dockerComposeFile": "../docker-compose.yml", + "customizations": { "vscode": { - "extensions": [ - "ms-python.python", - "ms-python.vscode-pylance", - "ms-toolsai.jupyter" - ] + "extensions": [ + "ms-python.python", + "ms-python.vscode-pylance", + "ms-toolsai.jupyter" + ] } - }, - - "forwardPorts": [3000] - } + }, + "forwardPorts": [3000], + "containerEnv": { + "TARGET": "base" +} +} \ No newline at end of file diff --git a/docker-compose.yaml b/docker-compose.yaml index 47563dd..38d3b97 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -4,9 +4,13 @@ services: build: context: . dockerfile: Dockerfile - target: test + target: $TARGET args: - - MAKE=tests + - TARGET=test + - MAKE=${MAKE:-tests} # Default value is "tests" if MAKE is not set + environment: + - TARGET=${TARGET} + - MAKE=${MAKE} selenium-chrome: image: selenium/standalone-chrome:latest From aee0c91d72f38c348b0e00fd819fdb474d794448 Mon Sep 17 00:00:00 2001 From: Erlich Sefi Date: Mon, 21 Aug 2023 20:57:23 +0000 Subject: [PATCH 15/47] . --- .devcontainer/devcontainer.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index e73c4e0..90719af 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,5 +1,7 @@ { "dockerComposeFile": "../docker-compose.yml", + "service":"chromegpt", + "workspaceFolder":"${workspaceFolder}", "customizations": { "vscode": { "extensions": [ From 61caf14a7d071765859960b6b77028cae9ec3a38 Mon Sep 17 00:00:00 2001 From: Erlich Sefi Date: Tue, 22 Aug 2023 13:04:11 +0300 Subject: [PATCH 16/47] Update docker-compose.yaml --- docker-compose.yaml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 38d3b97..d59e9be 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -4,17 +4,19 @@ services: build: context: . dockerfile: Dockerfile - target: $TARGET + target: ${TARGET:-test} args: - - TARGET=test + - TARGET=${TARGET:-test} - MAKE=${MAKE:-tests} # Default value is "tests" if MAKE is not set environment: - - TARGET=${TARGET} - - MAKE=${MAKE} + - TARGET=${TARGET:-test} + - MAKE=${MAKE:-tests} selenium-chrome: image: selenium/standalone-chrome:latest ports: - "4444:4444" - "7900:7900" - shm_size: "2g" \ No newline at end of file + shm_size: "2g" + depends_on: + - chromegpt From 5c171988175fd92927491e82e5f8ac969e33dd4d Mon Sep 17 00:00:00 2001 From: Erlich Sefi Date: Tue, 22 Aug 2023 13:04:42 +0300 Subject: [PATCH 17/47] Update devcontainer.json --- .devcontainer/devcontainer.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 90719af..c47104a 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,7 +1,7 @@ { - "dockerComposeFile": "../docker-compose.yml", - "service":"chromegpt", - "workspaceFolder":"${workspaceFolder}", + "dockerComposeFile": "../docker-compose.yaml", + "service": "chromegpt", + "workspaceFolder":"/app", "customizations": { "vscode": { "extensions": [ @@ -14,5 +14,5 @@ "forwardPorts": [3000], "containerEnv": { "TARGET": "base" + } } -} \ No newline at end of file From 7f964a20fd958dae9a495eec1b0bc49012d0cf47 Mon Sep 17 00:00:00 2001 From: Erlich Sefi Date: Tue, 22 Aug 2023 14:17:12 +0300 Subject: [PATCH 18/47] Update devcontainer.json --- .devcontainer/devcontainer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index c47104a..2c81474 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -11,7 +11,7 @@ ] } }, - "forwardPorts": [3000], + "forwardPorts": [3000,4444,7900], "containerEnv": { "TARGET": "base" } From 3feb48431f35b3250fa2e7c935914c4f344727f9 Mon Sep 17 00:00:00 2001 From: Erlich Sefi Date: Tue, 22 Aug 2023 14:32:40 +0300 Subject: [PATCH 19/47] Update devcontainer.json --- .devcontainer/devcontainer.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 2c81474..bd8fdb2 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -14,5 +14,6 @@ "forwardPorts": [3000,4444,7900], "containerEnv": { "TARGET": "base" - } + }, + "shutdownAction": "stopCompose" } From 04026f56ed333d1870344b2bf375bc88cd65349d Mon Sep 17 00:00:00 2001 From: Erlich Sefi Date: Tue, 22 Aug 2023 14:33:38 +0300 Subject: [PATCH 20/47] Update selenium.py --- chromegpt/tools/selenium.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/chromegpt/tools/selenium.py b/chromegpt/tools/selenium.py index 627df2a..19334a8 100644 --- a/chromegpt/tools/selenium.py +++ b/chromegpt/tools/selenium.py @@ -55,7 +55,9 @@ def __init__(self, headless: bool = False, docker: bool = True) -> None: def __del__(self) -> None: """Close Selenium session.""" - self.driver.close() + if hasattr(self,"driver") and self.driver != None: + self.driver.close() + self.driver.quit() def previous_webpage(self) -> str: """Go back in browser history.""" From 3c6ebf646d467caaa6cb7c2c258e7bd1df2fd40a Mon Sep 17 00:00:00 2001 From: Erlich Sefi Date: Tue, 22 Aug 2023 22:35:32 +0300 Subject: [PATCH 21/47] Update devcontainer.json --- .devcontainer/devcontainer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index bd8fdb2..73a6e89 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,7 +1,7 @@ { "dockerComposeFile": "../docker-compose.yaml", "service": "chromegpt", - "workspaceFolder":"/app", + "workspaceFolder":"/workspace", "customizations": { "vscode": { "extensions": [ From 064f55416df71e519e31a48d680464a0a95b5fd2 Mon Sep 17 00:00:00 2001 From: Erlich Sefi Date: Wed, 23 Aug 2023 20:40:44 +0300 Subject: [PATCH 22/47] Update devcontainer.json --- .devcontainer/devcontainer.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 73a6e89..cb9e64c 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,7 +1,7 @@ { "dockerComposeFile": "../docker-compose.yaml", "service": "chromegpt", - "workspaceFolder":"/workspace", + "workspaceFolder": "/workspace", "customizations": { "vscode": { "extensions": [ @@ -11,7 +11,7 @@ ] } }, - "forwardPorts": [3000,4444,7900], + "forwardPorts": [3000,4444,7900,5900], "containerEnv": { "TARGET": "base" }, From fbb4cfd5eda4200dc76074b63807a34d179549ae Mon Sep 17 00:00:00 2001 From: Erlich Sefi Date: Wed, 23 Aug 2023 20:41:23 +0300 Subject: [PATCH 23/47] Update lint.yml --- .github/workflows/lint.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 416c3f6..2628f2b 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -1,5 +1,7 @@ name: lint - +env: + TARGET: 'test' + MAKE: 'tests' on: push: branches: [main] @@ -22,7 +24,5 @@ jobs: docker 2>/dev/null 1>&2 rmi $(docker image ls --all --quiet) || true rm --recursive --force "$AGENT_TOOLSDIRECTORY" df --human-readable - - name: Build with Docker - run: docker build -t chromegpt --target test --build-arg MAKE=lint . - name: Test with pytest - run: docker run --rm --name test-run chromegpt + run: docker-compose up --abort-on-container-exit From 545f5a6334c892b59b9d83a10ca38254864732d2 Mon Sep 17 00:00:00 2001 From: Erlich Sefi Date: Wed, 23 Aug 2023 20:41:48 +0300 Subject: [PATCH 24/47] Update tests.yml --- .github/workflows/tests.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 2752440..fcac1a2 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,5 +1,7 @@ name: tests - +env: + TARGET: 'test' + MAKE: 'tests' on: push: branches: [main] @@ -22,7 +24,5 @@ jobs: docker 2>/dev/null 1>&2 rmi $(docker image ls --all --quiet) || true rm --recursive --force "$AGENT_TOOLSDIRECTORY" df --human-readable - - name: Build with Docker - run: docker build -t chromegpt --target test --build-arg MAKE=tests . - name: Test with pytest - run: docker run --rm --name test-run chromegpt + run: docker-compose up --abort-on-container-exit From 0bae0fa20b972bad51caad595aaf0ae47a66c37a Mon Sep 17 00:00:00 2001 From: Erlich Sefi Date: Wed, 23 Aug 2023 20:42:09 +0300 Subject: [PATCH 25/47] Update selenium.py --- chromegpt/tools/selenium.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/chromegpt/tools/selenium.py b/chromegpt/tools/selenium.py index 19334a8..8e8eb6a 100644 --- a/chromegpt/tools/selenium.py +++ b/chromegpt/tools/selenium.py @@ -40,14 +40,13 @@ class SeleniumWrapper: def __init__(self, headless: bool = False, docker: bool = True) -> None: """Initialize Selenium and start interactive session.""" chrome_options = Options() - if headless: chrome_options.add_argument("--headless") else: chrome_options.add_argument("--start-maximized") if docker: self.driver = webdriver.Remote( - "http://host.docker.internal:4444/wd/hub", options=chrome_options + "http://selenium-chrome:4444/wd/hub",options=chrome_options, ) else: self.driver = webdriver.Chrome(options=chrome_options) @@ -58,6 +57,7 @@ def __del__(self) -> None: if hasattr(self,"driver") and self.driver != None: self.driver.close() self.driver.quit() + def previous_webpage(self) -> str: """Go back in browser history.""" @@ -312,7 +312,7 @@ def fill_out_form(self, form_input: Optional[str] = None, **kwargs: Any) -> str: " website did not change after filling out form." ) except WebDriverException as e: - print(e) + #print(e) return f"Error filling out form with input {form_input}, message: {e.msg}" def scroll(self, direction: str) -> str: From 6294b54fd38f1515174b3aef71002d6a103146da Mon Sep 17 00:00:00 2001 From: Erlich Sefi Date: Wed, 23 Aug 2023 20:42:36 +0300 Subject: [PATCH 26/47] Update test_selenium.py --- tests/test_selenium.py | 73 +++++++++++++++++++++++++++--------------- 1 file changed, 47 insertions(+), 26 deletions(-) diff --git a/tests/test_selenium.py b/tests/test_selenium.py index 49a91dc..3aa88ce 100644 --- a/tests/test_selenium.py +++ b/tests/test_selenium.py @@ -1,47 +1,68 @@ """Integration test for Selenium API Wrapper.""" -import pytest +# import pytest from chromegpt.tools.selenium import SeleniumWrapper -@pytest.fixture -def client() -> SeleniumWrapper: - return SeleniumWrapper(headless=True) +# @pytest.fixture +# def client() -> SeleniumWrapper: +# return SeleniumWrapper(headless=True) -def test_describe_website(client: SeleniumWrapper) -> None: +def test_describe_website() -> None: """Test that SeleniumWrapper returns correct website""" + client = SeleniumWrapper(headless=True) + output = None + try: + output = client.describe_website("https://example.com") + finally: + del client + assert output is not None and "this domain is for use in illu" in output - output = client.describe_website("https://example.com") - assert "this domain is for use in illu" in output - -def test_click(client: SeleniumWrapper) -> None: +def test_click() -> None: """Test that SeleniumWrapper click works""" - - client.describe_website("https://example.com") - output = client.click_button_by_text('link with title "More information..."') - assert "Clicked interactable element and the website changed" in output + client = SeleniumWrapper(headless=True) + output = None + try: + client.describe_website("https://example.com") + output = client.click_button_by_text('link with title "More information..."') + finally: + del client + assert output is not None and "Clicked interactable element and the website changed" in output -def test_google_input(client: SeleniumWrapper) -> None: +def test_google_input() -> None: """Test that SeleniumWrapper can find input form""" + client = SeleniumWrapper(headless=True) + output = None + try: + output = client.find_form_inputs("https://google.com") + finally: + del client + assert output is not None and "q" in output - output = client.find_form_inputs("https://google.com") - assert "q" in output - -def test_google_fill(client: SeleniumWrapper) -> None: +def test_google_fill() -> None: """Test that SeleniumWrapper can fill input form""" - - client.find_form_inputs("https://google.com") - output = client.fill_out_form(q="hello world") - assert "website changed after filling out form" in output + client = SeleniumWrapper(headless=True) + output = None + try: + client.find_form_inputs("https://google.com") + output = client.fill_out_form(q="hello world") + finally: + del client + assert output is not None and "website changed after filling out form" in output -def test_google_search(client: SeleniumWrapper) -> None: +def test_google_search() -> None: """Test google search functionality""" - res = client.google_search("hello world") - assert "hello" in res - assert "Which url would you like to goto" in res + client = SeleniumWrapper(headless=True) + output = None + try: + output = client.google_search("hello world") + finally: + del client + assert output is not None and "hello" in output + assert output is not None and "Which url would you like to goto" in output From 9a5eef1aefef0147a531af35be230a9fa61a007c Mon Sep 17 00:00:00 2001 From: Erlich Sefi Date: Wed, 23 Aug 2023 22:58:49 +0300 Subject: [PATCH 27/47] Update lint.yml --- .github/workflows/lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 2628f2b..76cfd71 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -1,7 +1,7 @@ name: lint env: TARGET: 'test' - MAKE: 'tests' + MAKE: 'lint' on: push: branches: [main] From 2faf57ca51a46861aa5f226b53591bcb19aef56b Mon Sep 17 00:00:00 2001 From: Erlich Sefi Date: Thu, 24 Aug 2023 13:42:29 +0300 Subject: [PATCH 28/47] Update devcontainer.json --- .devcontainer/devcontainer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index cb9e64c..6d03dec 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,7 +1,7 @@ { "dockerComposeFile": "../docker-compose.yaml", "service": "chromegpt", - "workspaceFolder": "/workspace", + "workspaceFolder": "/app", "customizations": { "vscode": { "extensions": [ From ba0a0f089d2257d7e6605979f7611111a3c4329a Mon Sep 17 00:00:00 2001 From: Erlich Sefi Date: Thu, 24 Aug 2023 13:45:41 +0300 Subject: [PATCH 29/47] Update docker-compose.yaml --- docker-compose.yaml | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index d59e9be..c6ed72f 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -11,12 +11,23 @@ services: environment: - TARGET=${TARGET:-test} - MAKE=${MAKE:-tests} + depends_on: + selenium-chrome: + condition: service_healthy + selenium-chrome: image: selenium/standalone-chrome:latest ports: - "4444:4444" - "7900:7900" + - "5900:5900" + environment: + - SE_NODE_MAX_SESSIONS=10 + - SE_NODE_SESSION_TIMEOUT=30000 shm_size: "2g" - depends_on: - - chromegpt + healthcheck: + test: ["CMD", "curl", "-f", "http://selenium-chrome:4444/wd/hub/status"] + interval: 10s # Check every 30 seconds + timeout: 5s # Timeout after 10 seconds + retries: 3 # Retry 3 times before considering the container unhealthy From 391e929fca4611d2fc99387e2cb6dd62a23f990c Mon Sep 17 00:00:00 2001 From: Erlich Sefi Date: Fri, 25 Aug 2023 13:50:54 +0300 Subject: [PATCH 30/47] Create dev.docker-compose.yaml --- .devcontainer/dev.docker-compose.yaml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 .devcontainer/dev.docker-compose.yaml diff --git a/.devcontainer/dev.docker-compose.yaml b/.devcontainer/dev.docker-compose.yaml new file mode 100644 index 0000000..754b3f0 --- /dev/null +++ b/.devcontainer/dev.docker-compose.yaml @@ -0,0 +1,13 @@ +version: '3' +services: + chromegpt: + extends: + file: ../test.docker-compose.yaml + service: chromegpt + command: /bin/sh -c "while sleep 1000; do :; done" + build: + target: test + selenium-chrome: + extends: + file: ../test.docker-compose.yaml + service: selenium-chrome From bd08d33c4c8bf85a7305fa1687d6df35843a8194 Mon Sep 17 00:00:00 2001 From: Erlich Sefi Date: Fri, 25 Aug 2023 13:51:21 +0300 Subject: [PATCH 31/47] Rename docker-compose.yaml to test.docker-compose.yaml --- docker-compose.yaml => test.docker-compose.yaml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docker-compose.yaml => test.docker-compose.yaml (100%) diff --git a/docker-compose.yaml b/test.docker-compose.yaml similarity index 100% rename from docker-compose.yaml rename to test.docker-compose.yaml From ef25b8903a0693ca7e8f22c23c2be06d3411d852 Mon Sep 17 00:00:00 2001 From: Erlich Sefi Date: Fri, 25 Aug 2023 13:51:49 +0300 Subject: [PATCH 32/47] Update devcontainer.json --- .devcontainer/devcontainer.json | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 6d03dec..2edc87e 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,5 +1,5 @@ { - "dockerComposeFile": "../docker-compose.yaml", + "dockerComposeFile": "./dev.docker-compose.yaml", "service": "chromegpt", "workspaceFolder": "/app", "customizations": { @@ -12,8 +12,5 @@ } }, "forwardPorts": [3000,4444,7900,5900], - "containerEnv": { - "TARGET": "base" - }, "shutdownAction": "stopCompose" } From f0cb593fa169f4d6baea48bce8c56274bdaaae4a Mon Sep 17 00:00:00 2001 From: Erlich Sefi Date: Tue, 29 Aug 2023 18:33:43 +0000 Subject: [PATCH 33/47] add depends --- .devcontainer/dev.docker-compose.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.devcontainer/dev.docker-compose.yaml b/.devcontainer/dev.docker-compose.yaml index 754b3f0..1abb905 100644 --- a/.devcontainer/dev.docker-compose.yaml +++ b/.devcontainer/dev.docker-compose.yaml @@ -7,6 +7,9 @@ services: command: /bin/sh -c "while sleep 1000; do :; done" build: target: test + depends_on: + selenium-chrome: + condition: service_healthy selenium-chrome: extends: file: ../test.docker-compose.yaml From 167ddc48d4ab7c7f749f04a9be2f9cfdcffe73bf Mon Sep 17 00:00:00 2001 From: Erlich Sefi Date: Wed, 30 Aug 2023 00:04:13 +0300 Subject: [PATCH 34/47] Update devcontainer.json --- .devcontainer/devcontainer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 2edc87e..e95af47 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,5 +1,5 @@ { - "dockerComposeFile": "./dev.docker-compose.yaml", + "dockerComposeFile": "../docker-compose.yaml", "service": "chromegpt", "workspaceFolder": "/app", "customizations": { From ed30831ae7c08dc7626a7c491c5db6b95bcc0834 Mon Sep 17 00:00:00 2001 From: Erlich Sefi Date: Wed, 30 Aug 2023 00:04:39 +0300 Subject: [PATCH 35/47] Update and rename test.docker-compose.yaml to docker-compose.yaml --- test.docker-compose.yaml => docker-compose.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) rename test.docker-compose.yaml => docker-compose.yaml (78%) diff --git a/test.docker-compose.yaml b/docker-compose.yaml similarity index 78% rename from test.docker-compose.yaml rename to docker-compose.yaml index c6ed72f..12a470b 100644 --- a/test.docker-compose.yaml +++ b/docker-compose.yaml @@ -4,13 +4,13 @@ services: build: context: . dockerfile: Dockerfile - target: ${TARGET:-test} + target: ${TARGET:-dev} args: - - TARGET=${TARGET:-test} - - MAKE=${MAKE:-tests} # Default value is "tests" if MAKE is not set + - TARGET=${TARGET:-dev} + - MAKE=${MAKE} # Default value is "tests" if MAKE is not set environment: - - TARGET=${TARGET:-test} - - MAKE=${MAKE:-tests} + - TARGET=${TARGET:-dev} + - MAKE=${MAKE} depends_on: selenium-chrome: condition: service_healthy From 008f35bd0cce88c837dee10528889954d16cf3e3 Mon Sep 17 00:00:00 2001 From: Erlich Sefi Date: Wed, 30 Aug 2023 00:04:53 +0300 Subject: [PATCH 36/47] Update Dockerfile --- Dockerfile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index d4eae8a..41f5082 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,5 @@ FROM python:3.8 as base +RUN echo "building base." WORKDIR /app @@ -10,9 +11,12 @@ RUN pip install poetry==1.4.2 RUN poetry config virtualenvs.create false && \ poetry install --no-interaction --no-ansi +FROM base as dev +RUN echo "building dev." +CMD sh -c "while sleep 1000; do :; done" # run test FROM base as test # Set the environment variable ARG MAKE="tests" -CMD make $MAKE \ No newline at end of file +CMD make $MAKE From 1e9a6122e69670ede87e873f74dc710a43af48a0 Mon Sep 17 00:00:00 2001 From: Erlich Sefi Date: Tue, 29 Aug 2023 21:18:22 +0000 Subject: [PATCH 37/47] . --- .devcontainer/dev.docker-compose.yaml | 16 ---------------- .vscode/settings.json | 7 +++++++ chromegpt/tools/selenium.py | 8 ++++---- tests/test_selenium.py | 5 ++++- 4 files changed, 15 insertions(+), 21 deletions(-) delete mode 100644 .devcontainer/dev.docker-compose.yaml create mode 100644 .vscode/settings.json diff --git a/.devcontainer/dev.docker-compose.yaml b/.devcontainer/dev.docker-compose.yaml deleted file mode 100644 index 1abb905..0000000 --- a/.devcontainer/dev.docker-compose.yaml +++ /dev/null @@ -1,16 +0,0 @@ -version: '3' -services: - chromegpt: - extends: - file: ../test.docker-compose.yaml - service: chromegpt - command: /bin/sh -c "while sleep 1000; do :; done" - build: - target: test - depends_on: - selenium-chrome: - condition: service_healthy - selenium-chrome: - extends: - file: ../test.docker-compose.yaml - service: selenium-chrome diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..3e99ede --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,7 @@ +{ + "python.testing.pytestArgs": [ + "." + ], + "python.testing.unittestEnabled": false, + "python.testing.pytestEnabled": true +} \ No newline at end of file diff --git a/chromegpt/tools/selenium.py b/chromegpt/tools/selenium.py index 8e8eb6a..2db8a1f 100644 --- a/chromegpt/tools/selenium.py +++ b/chromegpt/tools/selenium.py @@ -46,7 +46,8 @@ def __init__(self, headless: bool = False, docker: bool = True) -> None: chrome_options.add_argument("--start-maximized") if docker: self.driver = webdriver.Remote( - "http://selenium-chrome:4444/wd/hub",options=chrome_options, + "http://selenium-chrome:4444/wd/hub", + options=chrome_options, ) else: self.driver = webdriver.Chrome(options=chrome_options) @@ -54,10 +55,9 @@ def __init__(self, headless: bool = False, docker: bool = True) -> None: def __del__(self) -> None: """Close Selenium session.""" - if hasattr(self,"driver") and self.driver != None: + if hasattr(self, "driver") and self.driver != None: self.driver.close() self.driver.quit() - def previous_webpage(self) -> str: """Go back in browser history.""" @@ -312,7 +312,7 @@ def fill_out_form(self, form_input: Optional[str] = None, **kwargs: Any) -> str: " website did not change after filling out form." ) except WebDriverException as e: - #print(e) + # print(e) return f"Error filling out form with input {form_input}, message: {e.msg}" def scroll(self, direction: str) -> str: diff --git a/tests/test_selenium.py b/tests/test_selenium.py index 3aa88ce..f599b4f 100644 --- a/tests/test_selenium.py +++ b/tests/test_selenium.py @@ -30,7 +30,10 @@ def test_click() -> None: output = client.click_button_by_text('link with title "More information..."') finally: del client - assert output is not None and "Clicked interactable element and the website changed" in output + assert ( + output is not None + and "Clicked interactable element and the website changed" in output + ) def test_google_input() -> None: From c05575a53dcdd73a60a918caefd419ed76ce8c4d Mon Sep 17 00:00:00 2001 From: Erlich Sefi Date: Wed, 30 Aug 2023 11:31:00 +0000 Subject: [PATCH 38/47] clear code --- Dockerfile | 6 ++-- chromegpt/tools/driver.py | 12 +++++++ chromegpt/tools/selenium.py | 3 +- docker-compose.yaml | 17 ++++++---- tests/test_selenium.py | 64 ++++++++++++------------------------- 5 files changed, 46 insertions(+), 56 deletions(-) create mode 100644 chromegpt/tools/driver.py diff --git a/Dockerfile b/Dockerfile index 41f5082..f3bb5fe 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,4 @@ FROM python:3.8 as base -RUN echo "building base." WORKDIR /app @@ -11,12 +10,11 @@ RUN pip install poetry==1.4.2 RUN poetry config virtualenvs.create false && \ poetry install --no-interaction --no-ansi +# image to dev FROM base as dev -RUN echo "building dev." CMD sh -c "while sleep 1000; do :; done" -# run test +# image to run tests FROM base as test -# Set the environment variable ARG MAKE="tests" CMD make $MAKE diff --git a/chromegpt/tools/driver.py b/chromegpt/tools/driver.py new file mode 100644 index 0000000..ddca1ed --- /dev/null +++ b/chromegpt/tools/driver.py @@ -0,0 +1,12 @@ +from chromegpt.tools.selenium import SeleniumWrapper + +def execute_with_driver(test_function): + def wrapper(*args, **kwargs): + try: + client = SeleniumWrapper(headless=True) + result = test_function(*args, **kwargs, client=client) + finally: + # release the driver + del client + return result + return wrapper diff --git a/chromegpt/tools/selenium.py b/chromegpt/tools/selenium.py index 2db8a1f..7f5623d 100644 --- a/chromegpt/tools/selenium.py +++ b/chromegpt/tools/selenium.py @@ -78,7 +78,8 @@ def google_search(self, query: str) -> str: results = self._get_google_search_results() return ( "Which url would you like to goto? Provide the full url starting with http" - " or https to goto: " + json.dumps(results) + " or https to goto: " + + json.dumps(results) ) def _get_google_search_results(self) -> List[Dict[str, Any]]: diff --git a/docker-compose.yaml b/docker-compose.yaml index 12a470b..bc18d53 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -6,13 +6,15 @@ services: dockerfile: Dockerfile target: ${TARGET:-dev} args: - - TARGET=${TARGET:-dev} - - MAKE=${MAKE} # Default value is "tests" if MAKE is not set - environment: - - TARGET=${TARGET:-dev} - - MAKE=${MAKE} + #- TARGET=${TARGET:-dev} # Default value is "dev" + - MAKE=${MAKE} + # environment: + # - TARGET=${TARGET:-dev} + # - MAKE=${MAKE} depends_on: - selenium-chrome: + # make sure the the image will be created only after Selenium is stable + # otherwise, tests a run before Selenium is accessible + selenium-chrome: condition: service_healthy @@ -26,7 +28,8 @@ services: - SE_NODE_MAX_SESSIONS=10 - SE_NODE_SESSION_TIMEOUT=30000 shm_size: "2g" - healthcheck: + # define Selenium stable 'Healthy' only after it's accessible. + healthcheck: test: ["CMD", "curl", "-f", "http://selenium-chrome:4444/wd/hub/status"] interval: 10s # Check every 30 seconds timeout: 5s # Timeout after 10 seconds diff --git a/tests/test_selenium.py b/tests/test_selenium.py index f599b4f..b03e1a7 100644 --- a/tests/test_selenium.py +++ b/tests/test_selenium.py @@ -1,71 +1,47 @@ """Integration test for Selenium API Wrapper.""" -# import pytest +from chromegpt.tools.driver import execute_with_driver,SeleniumWrapper -from chromegpt.tools.selenium import SeleniumWrapper - -# @pytest.fixture -# def client() -> SeleniumWrapper: -# return SeleniumWrapper(headless=True) - - -def test_describe_website() -> None: +@execute_with_driver +def test_describe_website(client:SeleniumWrapper) -> None: """Test that SeleniumWrapper returns correct website""" - client = SeleniumWrapper(headless=True) output = None - try: - output = client.describe_website("https://example.com") - finally: - del client + output = client.describe_website("https://example.com") assert output is not None and "this domain is for use in illu" in output - -def test_click() -> None: +@execute_with_driver +def test_click(client:SeleniumWrapper) -> None: """Test that SeleniumWrapper click works""" - client = SeleniumWrapper(headless=True) + output = None - try: - client.describe_website("https://example.com") - output = client.click_button_by_text('link with title "More information..."') - finally: - del client + client.describe_website("https://example.com") + output = client.click_button_by_text('link with title "More information..."') assert ( output is not None and "Clicked interactable element and the website changed" in output ) - -def test_google_input() -> None: +@execute_with_driver +def test_google_input(client:SeleniumWrapper) -> None: """Test that SeleniumWrapper can find input form""" - client = SeleniumWrapper(headless=True) output = None - try: - output = client.find_form_inputs("https://google.com") - finally: - del client + output = client.find_form_inputs("https://google.com") assert output is not None and "q" in output - -def test_google_fill() -> None: +@execute_with_driver +def test_google_fill(client:SeleniumWrapper) -> None: """Test that SeleniumWrapper can fill input form""" - client = SeleniumWrapper(headless=True) output = None - try: - client.find_form_inputs("https://google.com") - output = client.fill_out_form(q="hello world") - finally: - del client - assert output is not None and "website changed after filling out form" in output + client.find_form_inputs("https://google.com") + output = client.fill_out_form(q="hello world") + assert output is not None and "website changed after filling out form" in output -def test_google_search() -> None: +@execute_with_driver +def test_google_search(client:SeleniumWrapper) -> None: """Test google search functionality""" - client = SeleniumWrapper(headless=True) output = None - try: - output = client.google_search("hello world") - finally: - del client + output = client.google_search("hello world") assert output is not None and "hello" in output assert output is not None and "Which url would you like to goto" in output From 894223dcfe429ae8b27b41c2ce8cd45b35f77752 Mon Sep 17 00:00:00 2001 From: Erlich Sefi Date: Wed, 30 Aug 2023 11:31:23 +0000 Subject: [PATCH 39/47] . --- chromegpt/tools/driver.py | 4 +++- tests/test_selenium.py | 18 +++++++++++------- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/chromegpt/tools/driver.py b/chromegpt/tools/driver.py index ddca1ed..840f3f0 100644 --- a/chromegpt/tools/driver.py +++ b/chromegpt/tools/driver.py @@ -1,12 +1,14 @@ from chromegpt.tools.selenium import SeleniumWrapper + def execute_with_driver(test_function): def wrapper(*args, **kwargs): try: client = SeleniumWrapper(headless=True) result = test_function(*args, **kwargs, client=client) finally: - # release the driver + # release the driver del client return result + return wrapper diff --git a/tests/test_selenium.py b/tests/test_selenium.py index b03e1a7..e9cf911 100644 --- a/tests/test_selenium.py +++ b/tests/test_selenium.py @@ -1,19 +1,20 @@ """Integration test for Selenium API Wrapper.""" -from chromegpt.tools.driver import execute_with_driver,SeleniumWrapper +from chromegpt.tools.driver import execute_with_driver, SeleniumWrapper @execute_with_driver -def test_describe_website(client:SeleniumWrapper) -> None: +def test_describe_website(client: SeleniumWrapper) -> None: """Test that SeleniumWrapper returns correct website""" output = None output = client.describe_website("https://example.com") assert output is not None and "this domain is for use in illu" in output + @execute_with_driver -def test_click(client:SeleniumWrapper) -> None: +def test_click(client: SeleniumWrapper) -> None: """Test that SeleniumWrapper click works""" - + output = None client.describe_website("https://example.com") output = client.click_button_by_text('link with title "More information..."') @@ -22,15 +23,17 @@ def test_click(client:SeleniumWrapper) -> None: and "Clicked interactable element and the website changed" in output ) + @execute_with_driver -def test_google_input(client:SeleniumWrapper) -> None: +def test_google_input(client: SeleniumWrapper) -> None: """Test that SeleniumWrapper can find input form""" output = None output = client.find_form_inputs("https://google.com") assert output is not None and "q" in output + @execute_with_driver -def test_google_fill(client:SeleniumWrapper) -> None: +def test_google_fill(client: SeleniumWrapper) -> None: """Test that SeleniumWrapper can fill input form""" output = None client.find_form_inputs("https://google.com") @@ -38,8 +41,9 @@ def test_google_fill(client:SeleniumWrapper) -> None: assert output is not None and "website changed after filling out form" in output + @execute_with_driver -def test_google_search(client:SeleniumWrapper) -> None: +def test_google_search(client: SeleniumWrapper) -> None: """Test google search functionality""" output = None output = client.google_search("hello world") From d32fc2ea514c81a62bedcfc9119f9861b1cd1f15 Mon Sep 17 00:00:00 2001 From: Erlich Sefi Date: Wed, 30 Aug 2023 11:32:18 +0000 Subject: [PATCH 40/47] . --- docker-compose.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index bc18d53..878f648 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -4,9 +4,9 @@ services: build: context: . dockerfile: Dockerfile - target: ${TARGET:-dev} + target: ${TARGET:-dev} # Default value is "dev" args: - #- TARGET=${TARGET:-dev} # Default value is "dev" + #- TARGET=${TARGET:-dev} - MAKE=${MAKE} # environment: # - TARGET=${TARGET:-dev} From 89a6099fae3ba4ea032b107e10d480f58ea34c2e Mon Sep 17 00:00:00 2001 From: Erlich Sefi Date: Wed, 30 Aug 2023 11:48:08 +0000 Subject: [PATCH 41/47] click to dev --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 54dd9bc..19a74e5 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ [![lint](https://github.com/richardyc/chrome-gpt/actions/workflows/lint.yml/badge.svg)](https://github.com/richardyc/chrome-gpt/actions/workflows/lint.yml) [![test](https://github.com/richardyc/chrome-gpt/actions/workflows/tests.yml/badge.svg)](https://github.com/richardyc/chrome-gpt/actions/workflows/tests.yml) [![Twitter](https://img.shields.io/twitter/url/https/twitter.com/RealRichomie.svg?style=social&label=Follow%20%40RealRichomie)](https://twitter.com/RealRichomie) +[![Open in GitHub Codespaces](https://github.com/codespaces/badge.svg)](https://github.com/erlichsefi/Chrome-GPT/codespaces) + ⚠️This is an experimental AutoGPT agent that might take incorrect actions and could lead to serious consequences. Please use it at your own discretion⚠️ Chrome-GPT is an AutoGPT experiment that utilizes [Langchain](https://github.com/hwchase17/langchain) and [Selenium](https://github.com/SeleniumHQ/selenium) to enable an AutoGPT agent take control of an entire Chrome session. With the ability to interactively scroll, click, and input text on web pages, the AutoGPT agent can navigate and manipulate web content. From 23d003f06c3cd364767cd811bf29b180b4e714b0 Mon Sep 17 00:00:00 2001 From: Erlich Sefi Date: Wed, 30 Aug 2023 11:54:17 +0000 Subject: [PATCH 42/47] . --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index f3bb5fe..564c387 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,6 +10,8 @@ RUN pip install poetry==1.4.2 RUN poetry config virtualenvs.create false && \ poetry install --no-interaction --no-ansi +CMD python -m chromegpt -v -t $REQUEST + # image to dev FROM base as dev CMD sh -c "while sleep 1000; do :; done" From 1b0466151c64a45fd78c176c3c99dcf4482d94fb Mon Sep 17 00:00:00 2001 From: Erlich Sefi Date: Wed, 30 Aug 2023 12:04:38 +0000 Subject: [PATCH 43/47] formating --- chromegpt/tools/selenium.py | 2 +- chromegpt/tools/utils.py | 2 +- tests/test_selenium.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/chromegpt/tools/selenium.py b/chromegpt/tools/selenium.py index 7f5623d..cf908f4 100644 --- a/chromegpt/tools/selenium.py +++ b/chromegpt/tools/selenium.py @@ -55,7 +55,7 @@ def __init__(self, headless: bool = False, docker: bool = True) -> None: def __del__(self) -> None: """Close Selenium session.""" - if hasattr(self, "driver") and self.driver != None: + if hasattr(self, "driver") and self.driver is not None: self.driver.close() self.driver.quit() diff --git a/chromegpt/tools/utils.py b/chromegpt/tools/utils.py index d42b784..3285058 100644 --- a/chromegpt/tools/utils.py +++ b/chromegpt/tools/utils.py @@ -4,8 +4,8 @@ from typing import List, Optional, Union from selenium.webdriver.chrome.webdriver import WebDriver -from selenium.webdriver.remote.webdriver import WebDriver as RemoteWebDriver from selenium.webdriver.common.by import By +from selenium.webdriver.remote.webdriver import WebDriver as RemoteWebDriver from selenium.webdriver.remote.webelement import WebElement from unidecode import unidecode diff --git a/tests/test_selenium.py b/tests/test_selenium.py index e9cf911..b3d61fb 100644 --- a/tests/test_selenium.py +++ b/tests/test_selenium.py @@ -1,6 +1,6 @@ """Integration test for Selenium API Wrapper.""" -from chromegpt.tools.driver import execute_with_driver, SeleniumWrapper +from chromegpt.tools.driver import SeleniumWrapper, execute_with_driver @execute_with_driver From 210866d7c0de1766c52d5db85f4147ab982ceccf Mon Sep 17 00:00:00 2001 From: Erlich Sefi Date: Wed, 30 Aug 2023 12:51:09 +0000 Subject: [PATCH 44/47] foramting --- chromegpt/tools/driver.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/chromegpt/tools/driver.py b/chromegpt/tools/driver.py index 840f3f0..f65d539 100644 --- a/chromegpt/tools/driver.py +++ b/chromegpt/tools/driver.py @@ -1,14 +1,15 @@ +from typing import Any, Callable + from chromegpt.tools.selenium import SeleniumWrapper -def execute_with_driver(test_function): - def wrapper(*args, **kwargs): +def execute_with_driver(test_function: Callable[[SeleniumWrapper], None]) -> Callable: + def wrapper(*args: Any, **kwargs: Any) -> None: try: client = SeleniumWrapper(headless=True) - result = test_function(*args, **kwargs, client=client) + test_function(client, *args, **kwargs) finally: # release the driver del client - return result return wrapper From cc9eb58477d77bed8ecd25a62888f5a2984e0b55 Mon Sep 17 00:00:00 2001 From: Erlich Sefi Date: Wed, 30 Aug 2023 18:09:31 +0000 Subject: [PATCH 45/47] add example .env --- .env | 3 +++ .gitignore | 1 - Dockerfile | 2 +- docker-compose.yaml | 7 +++---- 4 files changed, 7 insertions(+), 6 deletions(-) create mode 100644 .env diff --git a/.env b/.env new file mode 100644 index 0000000..2eca185 --- /dev/null +++ b/.env @@ -0,0 +1,3 @@ +OPENAI_API_KEY=sk- +REQUEST="Find me a bar that can host a 20 person event near Chelsea, Manhattan evening of Apr 30th. Fill out contact us form if they have one with info: Name Richard, email he@hrichard.com." +TARGET=base \ No newline at end of file diff --git a/.gitignore b/.gitignore index d9005f2..42d6935 100644 --- a/.gitignore +++ b/.gitignore @@ -112,7 +112,6 @@ celerybeat.pid *.sage.py # Environments -.env .venv env/ venv/ diff --git a/Dockerfile b/Dockerfile index 564c387..2dbaed5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,7 +10,7 @@ RUN pip install poetry==1.4.2 RUN poetry config virtualenvs.create false && \ poetry install --no-interaction --no-ansi -CMD python -m chromegpt -v -t $REQUEST +CMD python -m chromegpt -v -t "${REQUEST}" # image to dev FROM base as dev diff --git a/docker-compose.yaml b/docker-compose.yaml index 878f648..d640779 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -6,11 +6,10 @@ services: dockerfile: Dockerfile target: ${TARGET:-dev} # Default value is "dev" args: - #- TARGET=${TARGET:-dev} - MAKE=${MAKE} - # environment: - # - TARGET=${TARGET:-dev} - # - MAKE=${MAKE} + environment: + - OPENAI_API_KEY=${OPENAI_API_KEY} + - REQUEST=${REQUEST} depends_on: # make sure the the image will be created only after Selenium is stable # otherwise, tests a run before Selenium is accessible From 05d481087558f371cdbc0ba8d2c40f0166ec7112 Mon Sep 17 00:00:00 2001 From: Erlich Sefi Date: Wed, 30 Aug 2023 18:09:49 +0000 Subject: [PATCH 46/47] remove and .env updates --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 42d6935..d9005f2 100644 --- a/.gitignore +++ b/.gitignore @@ -112,6 +112,7 @@ celerybeat.pid *.sage.py # Environments +.env .venv env/ venv/ From b360ecaec8ca6a2b4feba5403052d4bfc9d14d48 Mon Sep 17 00:00:00 2001 From: Erlich Sefi Date: Wed, 30 Aug 2023 18:18:54 +0000 Subject: [PATCH 47/47] . --- README.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 19a74e5..ccde9a2 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,6 @@ [![lint](https://github.com/richardyc/chrome-gpt/actions/workflows/lint.yml/badge.svg)](https://github.com/richardyc/chrome-gpt/actions/workflows/lint.yml) [![test](https://github.com/richardyc/chrome-gpt/actions/workflows/tests.yml/badge.svg)](https://github.com/richardyc/chrome-gpt/actions/workflows/tests.yml) [![Twitter](https://img.shields.io/twitter/url/https/twitter.com/RealRichomie.svg?style=social&label=Follow%20%40RealRichomie)](https://twitter.com/RealRichomie) -[![Open in GitHub Codespaces](https://github.com/codespaces/badge.svg)](https://github.com/erlichsefi/Chrome-GPT/codespaces) ⚠️This is an experimental AutoGPT agent that might take incorrect actions and could lead to serious consequences. Please use it at your own discretion⚠️ @@ -43,6 +42,11 @@ Demo made by [Richard He](https://twitter.com/RealRichomie) 3. Open a poetry shell `poetry shell` 4. Run chromegpt via `python -m chromegpt` + +You can start in you own codespace here: + +[![Open in GitHub Codespaces](https://github.com/codespaces/badge.svg)](https://github.com/erlichsefi/Chrome-GPT/codespaces) +

🧠 Usage

- GPT-3.5 Usage (Default): `python -m chromegpt -v -t "{your request}"` @@ -65,6 +69,10 @@ Options: --help Show this message and exit. ``` +Or Just update .env and + +`source .env & docker-compose up` +

⭐ Star History

[![Star History Chart](https://api.star-history.com/svg?repos=richardyc/Chrome-GPT&type=Date)](https://star-history.com/#richardyc/Chrome-GPT&Date)