From cbccd49d7599cbd1b02a9d49dfb2ce48cbc6766d Mon Sep 17 00:00:00 2001 From: Shankar Pandala Date: Sun, 26 Nov 2023 16:08:51 +0530 Subject: [PATCH] Add new files and dependencies --- .gitignore | 4 + folderstructure.py | 16 +++ lazygitgpt/agents/__init__.py | 1 + lazygitgpt/agents/cli_agent.py | 30 +++++ lazygitgpt/ai_operations.py | 87 -------------- lazygitgpt/cli.py | 4 +- lazygitgpt/datasources/__init__.py | 1 + lazygitgpt/datasources/repos.py | 26 ++++ lazygitgpt/git/__init__.py | 1 + .../{git_operations.py => git/operations.py} | 113 ++++++++++-------- lazygitgpt/llms/__init__.py | 1 + lazygitgpt/llms/openai.py | 3 + lazygitgpt/memory/__init__.py | 0 lazygitgpt/parsers/__init__.py | 0 lazygitgpt/prompts/__init__.py | 0 lazygitgpt/retrievers/__init__.py | 0 lazygitgpt/utils/__init__.py | 0 lazygitgpt/vectorstores/__init__.py | 0 requirements.txt | 6 +- 19 files changed, 153 insertions(+), 140 deletions(-) create mode 100644 folderstructure.py create mode 100644 lazygitgpt/agents/__init__.py create mode 100644 lazygitgpt/agents/cli_agent.py delete mode 100644 lazygitgpt/ai_operations.py create mode 100644 lazygitgpt/datasources/__init__.py create mode 100644 lazygitgpt/datasources/repos.py create mode 100644 lazygitgpt/git/__init__.py rename lazygitgpt/{git_operations.py => git/operations.py} (81%) create mode 100644 lazygitgpt/llms/__init__.py create mode 100644 lazygitgpt/llms/openai.py create mode 100644 lazygitgpt/memory/__init__.py create mode 100644 lazygitgpt/parsers/__init__.py create mode 100644 lazygitgpt/prompts/__init__.py create mode 100644 lazygitgpt/retrievers/__init__.py create mode 100644 lazygitgpt/utils/__init__.py create mode 100644 lazygitgpt/vectorstores/__init__.py diff --git a/.gitignore b/.gitignore index 51e9d59..db17736 100644 --- a/.gitignore +++ b/.gitignore @@ -158,3 +158,7 @@ cython_debug/ # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ + +*.pkl +*.bin +*.ipynb \ No newline at end of file diff --git a/folderstructure.py b/folderstructure.py new file mode 100644 index 0000000..a52a132 --- /dev/null +++ b/folderstructure.py @@ -0,0 +1,16 @@ +import os +from pathlib import Path + +def create_folder_structure_md(path, indent=0): + markdown = "" + path = Path(path) + if path.is_file() and not path.name.startswith('.') and path.name != 'folderstructure.py': + markdown += " " * indent + "- " + path.name + "\n" + elif path.is_dir() and not path.name.startswith('.') and path.name != 'folderstructure.py': + markdown += " " * indent + "- " + path.name + "/\n" + for child in path.iterdir(): + markdown += create_folder_structure_md(child, indent + 1) + return markdown + +# Replace 'your_directory_path' with the path of the directory you want to convert to markdown +print(create_folder_structure_md(os.getcwd())) diff --git a/lazygitgpt/agents/__init__.py b/lazygitgpt/agents/__init__.py new file mode 100644 index 0000000..cee7659 --- /dev/null +++ b/lazygitgpt/agents/__init__.py @@ -0,0 +1 @@ +from .cli_agent import generate_response \ No newline at end of file diff --git a/lazygitgpt/agents/cli_agent.py b/lazygitgpt/agents/cli_agent.py new file mode 100644 index 0000000..17c8a29 --- /dev/null +++ b/lazygitgpt/agents/cli_agent.py @@ -0,0 +1,30 @@ +import json +from langchain.prompts import ChatPromptTemplate +from langchain.output_parsers import ResponseSchema, StructuredOutputParser +import re + +from lazygitgpt.llms import chat_model +from lazygitgpt.datasources.repos import read_repository_contents +from lazygitgpt.git.operations import update_files + +output_schema = ResponseSchema(name='filename', description='contents', type='string') +output_parser = StructuredOutputParser(response_schemas=[output_schema]) +format_instructions = output_parser.get_format_instructions() +template_string = """You are an expert programmer. +You are reviewing a code repository. +Read the code and make changes to the code as per the user requirements. +user requirements: {user_requirements} +code repository: {code_repository} +Output the contents of the file that you changed as per the format instructions : {format_instructions} +""" + +def generate_response(prompt, sources=read_repository_contents()): + sources_str = json.dumps(sources, indent=4) + prompt_template = ChatPromptTemplate.from_template(template_string) + messages = prompt_template.format_messages(user_requirements = prompt, + code_repository = sources_str, + format_instructions=format_instructions) + response = chat_model(messages) + response_json = response.to_json() + data = response_json['kwargs']['content'] + return data diff --git a/lazygitgpt/ai_operations.py b/lazygitgpt/ai_operations.py deleted file mode 100644 index af5c4a4..0000000 --- a/lazygitgpt/ai_operations.py +++ /dev/null @@ -1,87 +0,0 @@ -import os -import json -from tqdm import tqdm -from langchain.chat_models import ChatOpenAI -from langchain.prompts import ChatPromptTemplate -from langchain.schema import HumanMessage -from langchain.output_parsers import ResponseSchema, StructuredOutputParser -import re - -# Set environment variables -os.environ["LANGCHAIN_TRACING_V2"] = "true" -os.environ["LANGCHAIN_ENDPOINT"] = "https://api.smith.langchain.com" -os.environ["LANGCHAIN_PROJECT"] = "lazygitgpt" -os.environ["LANGCHAIN_API_KEY"] = "ls__677090643195489ea72be5d7f03d6fbf" - -chat_model = ChatOpenAI(model="gpt-4-1106-preview", temperature=0.0) -output_schema = ResponseSchema(name='filename', description='contents', type='string') -output_parser = StructuredOutputParser(response_schemas=[output_schema]) -format_instructions = output_parser.get_format_instructions() -template_string = """You are an expert programmer. -You are reviewing a code repository. -Read the code and make changes to the code as per the user requirements. -user requirements: {user_requirements} -code repository: {code_repository} -Output the contents of the file that you changed as per the format instructions : {format_instructions} -""" -# {format_instructions} -def is_text_file(filepath): - text_file_extensions = ['.txt', '.py', '.html', '.css', '.js', '.md', '.json', '.xml'] - return os.path.splitext(filepath)[1].lower() in text_file_extensions - -def read_repository_contents(): - current_directory = os.getcwd() - file_contents_dict = {} - file_list = os.listdir(current_directory) - - for filename in tqdm(file_list, desc="Reading files", unit="file"): - file_path = os.path.join(current_directory, filename) - - if not is_text_file(file_path): - continue # Skip non-text files - - # Read file contents - try: - with open(file_path, 'r', encoding='utf-8') as file: - file_contents = file.read() - file_contents_dict[filename] = file_contents - except Exception as e: - print(f"Error reading file {filename}: {e}") - - return file_contents_dict - -def update_files(response): - try: - for filename, contents in response.items(): - file_path = os.path.join(os.getcwd(), filename) - - with open(file_path, 'w', encoding='utf-8') as file: - file.write(contents) - - print(f"Updated file: {filename}") - - except Exception as e: - print(f"Error updating files: {e}") - - - -def extract_json_data(data): - json_data = re.search(r'```json(.*)```', data, re.DOTALL) - if json_data: - return json_data.group(1).strip() - else: - return None - -def generate_response(prompt, sources=read_repository_contents()): - sources_str = json.dumps(sources, indent=4) - prompt_template = ChatPromptTemplate.from_template(template_string) - messages = prompt_template.format_messages(user_requirements = prompt, - code_repository = sources_str, - format_instructions=format_instructions) - response = chat_model(messages) - response_json = response.to_json() - data = response_json['kwargs']['content'] - data = extract_json_data(data) - data = json.loads(data) - update_files(data) - return data diff --git a/lazygitgpt/cli.py b/lazygitgpt/cli.py index bfe02b1..f7c40f9 100644 --- a/lazygitgpt/cli.py +++ b/lazygitgpt/cli.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 import click -from lazygitgpt.git_operations import clone_repository, checkout_branch, create_branch -from lazygitgpt.ai_operations import generate_response +from .git.operations import clone_repository, checkout_branch, create_branch +from .agents.cli_agent import generate_response @click.group() def cli(): diff --git a/lazygitgpt/datasources/__init__.py b/lazygitgpt/datasources/__init__.py new file mode 100644 index 0000000..5e6f4c4 --- /dev/null +++ b/lazygitgpt/datasources/__init__.py @@ -0,0 +1 @@ +from .repos import read_repository_contents \ No newline at end of file diff --git a/lazygitgpt/datasources/repos.py b/lazygitgpt/datasources/repos.py new file mode 100644 index 0000000..abb1a58 --- /dev/null +++ b/lazygitgpt/datasources/repos.py @@ -0,0 +1,26 @@ +import os +import glob +import json + +def read_repository_contents(directory_path=os.getcwd(), file_pattern="*"): + """ + Reads all files in the specified directory matching the file pattern, + and creates a JSON object with file names and their contents. + + Args: + directory_path (str): Path to the directory containing the files. + file_pattern (str): Pattern to match files. Defaults to '*' (all files). + + Returns: + str: A JSON string containing the file names and their contents. + """ + data = {} + for file_path in glob.glob(f"{directory_path}/{file_pattern}"): + if os.path.isfile(file_path): + try: + with open(file_path, 'r', encoding='utf-8') as file: + data[file_path] = file.read() + except Exception as e: + print(f"Error reading file: {file_path} - {e}") + + return json.dumps(data, indent=4) diff --git a/lazygitgpt/git/__init__.py b/lazygitgpt/git/__init__.py new file mode 100644 index 0000000..d13593f --- /dev/null +++ b/lazygitgpt/git/__init__.py @@ -0,0 +1 @@ +from .operations import update_files, clone_repository, checkout_branch, create_branch \ No newline at end of file diff --git a/lazygitgpt/git_operations.py b/lazygitgpt/git/operations.py similarity index 81% rename from lazygitgpt/git_operations.py rename to lazygitgpt/git/operations.py index e64ada7..e546c9f 100644 --- a/lazygitgpt/git_operations.py +++ b/lazygitgpt/git/operations.py @@ -1,51 +1,64 @@ -import os -from git import Repo, GitCommandError - -def clone_repository(repo_url): - """ - Clones a git repository from the given URL to the current working directory. - - :param repo_url: URL of the repository to clone. - """ - try: - # Get the name of the repository by parsing the URL - repo_name = os.path.basename(repo_url) - # If the URL ends with '.git', remove it - if repo_name.endswith('.git'): - repo_name = repo_name[:-4] - # Use the current working directory as the destination - destination = os.path.join(os.getcwd(), repo_name) - Repo.clone_from(repo_url, destination) - print(f"Repository cloned successfully to {destination}") - except GitCommandError as e: - print(f"Error cloning repository: {e}") - -def checkout_branch(branch_name): - """ - Checks out a specified branch in the current working directory. - - :param branch_name: Name of the branch to checkout. - """ - try: - # Use the current working directory as the repository path - repo_path = os.getcwd() - repo = Repo(repo_path) - repo.git.checkout(branch_name) - print(f"Checked out to branch {branch_name}") - except GitCommandError as e: - print(f"Error checking out branch: {e}") - -def create_branch(branch_name): - """ - Creates a new branch in the current working directory. - - :param branch_name: Name of the branch to create. - """ - try: - # Use the current working directory as the repository path - repo_path = os.getcwd() - repo = Repo(repo_path) - repo.git.checkout('-b', branch_name) - print(f"Created and checked out to new branch {branch_name}") - except GitCommandError as e: +import os +from git import Repo, GitCommandError + +def update_files(response): + try: + for filename, contents in response.items(): + file_path = os.path.join(os.getcwd(), filename) + + with open(file_path, 'w', encoding='utf-8') as file: + file.write(contents) + + print(f"Updated file: {filename}") + + except Exception as e: + print(f"Error updating files: {e}") + +def clone_repository(repo_url): + """ + Clones a git repository from the given URL to the current working directory. + + :param repo_url: URL of the repository to clone. + """ + try: + # Get the name of the repository by parsing the URL + repo_name = os.path.basename(repo_url) + # If the URL ends with '.git', remove it + if repo_name.endswith('.git'): + repo_name = repo_name[:-4] + # Use the current working directory as the destination + destination = os.path.join(os.getcwd(), repo_name) + Repo.clone_from(repo_url, destination) + print(f"Repository cloned successfully to {destination}") + except GitCommandError as e: + print(f"Error cloning repository: {e}") + +def checkout_branch(branch_name): + """ + Checks out a specified branch in the current working directory. + + :param branch_name: Name of the branch to checkout. + """ + try: + # Use the current working directory as the repository path + repo_path = os.getcwd() + repo = Repo(repo_path) + repo.git.checkout(branch_name) + print(f"Checked out to branch {branch_name}") + except GitCommandError as e: + print(f"Error checking out branch: {e}") + +def create_branch(branch_name): + """ + Creates a new branch in the current working directory. + + :param branch_name: Name of the branch to create. + """ + try: + # Use the current working directory as the repository path + repo_path = os.getcwd() + repo = Repo(repo_path) + repo.git.checkout('-b', branch_name) + print(f"Created and checked out to new branch {branch_name}") + except GitCommandError as e: print(f"Error creating new branch: {e}") \ No newline at end of file diff --git a/lazygitgpt/llms/__init__.py b/lazygitgpt/llms/__init__.py new file mode 100644 index 0000000..7a7150a --- /dev/null +++ b/lazygitgpt/llms/__init__.py @@ -0,0 +1 @@ +from .openai import chat_model \ No newline at end of file diff --git a/lazygitgpt/llms/openai.py b/lazygitgpt/llms/openai.py new file mode 100644 index 0000000..885a9f3 --- /dev/null +++ b/lazygitgpt/llms/openai.py @@ -0,0 +1,3 @@ +from langchain.chat_models import ChatOpenAI + +chat_model = ChatOpenAI(model="gpt-4-1106-preview", temperature=0.0) \ No newline at end of file diff --git a/lazygitgpt/memory/__init__.py b/lazygitgpt/memory/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/lazygitgpt/parsers/__init__.py b/lazygitgpt/parsers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/lazygitgpt/prompts/__init__.py b/lazygitgpt/prompts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/lazygitgpt/retrievers/__init__.py b/lazygitgpt/retrievers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/lazygitgpt/utils/__init__.py b/lazygitgpt/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/lazygitgpt/vectorstores/__init__.py b/lazygitgpt/vectorstores/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/requirements.txt b/requirements.txt index 331a505..5f1202c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,8 @@ gitpython>=3.1.0 openai>=0.10.0 click -langchain \ No newline at end of file +langchain +duckduckgo-search +llmx +cohere +tiktoken \ No newline at end of file