diff --git a/.DS_Store b/.DS_Store deleted file mode 100644 index e93d3ba..0000000 Binary files a/.DS_Store and /dev/null differ diff --git a/.gitignore b/.gitignore index c600a0b..5a00f89 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ env/* build/* cyberhead.egg-info/* dist/* +.DS_Store \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..392a4bf --- /dev/null +++ b/Dockerfile @@ -0,0 +1,13 @@ +FROM ubuntu:latest + +WORKDIR /app +ADD ./cyberhead /app + +RUN apt-get update +RUN apt-get install -y python3 python3-pip telnet curl make g++ libssl-dev git +RUN curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - +RUN echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list +RUN apt-get update +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get install -y yarn +RUN pip3 install -r ./requirements.txt diff --git a/cyberhead/database.py b/cyberhead/database.py index 9b2f236..9cc5510 100644 --- a/cyberhead/database.py +++ b/cyberhead/database.py @@ -3,7 +3,7 @@ import os -engine = create_engine(f"mysql+pymysql://root:root@localhost:3306/cyberhead").connect() +engine = create_engine(f"mysql+pymysql://{os.getenv('CH_DB_USER')}:{os.getenv('CH_DB_PASSWORD')}@{os.getenv('CH_DB_HOST')}:3306/{os.getenv('CH_DB_NAME')}").connect() db = peewee.MySQLDatabase(os.getenv('CH_DB_NAME'), host=os.getenv('CH_DB_HOST'), diff --git a/cyberhead/main.py b/cyberhead/main.py index 0ac4631..288bacd 100644 --- a/cyberhead/main.py +++ b/cyberhead/main.py @@ -143,4 +143,4 @@ def heatmap(): """ if __name__ == '__main__': run_loader() - app.run(debug=True) + app.run(debug=True, host='0.0.0.0') diff --git a/cyberhead/requirements.txt b/cyberhead/requirements.txt index eb00bfd..c2b3d64 100644 --- a/cyberhead/requirements.txt +++ b/cyberhead/requirements.txt @@ -1,4 +1,5 @@ amqp==2.5.2 +asn1crypto==0.24.0 Backtesting==0.1.7 billiard==3.6.3.0 bokeh==2.0.2 @@ -8,15 +9,21 @@ cffi==1.14.0 chardet==3.0.4 click==7.1.2 cryptography==2.9.2 +cycler==0.10.0 +dataclasses==0.6 Flask==1.1.2 Flask-Cors==3.0.8 idna==2.8 importlib-metadata==1.6.0 itsdangerous==1.1.0 Jinja2==2.11.2 +keyring==10.6.0 +keyrings.alt==3.0 +kiwisolver==1.2.0 kombu==4.6.8 lxml==4.5.0 MarkupSafe==1.1.1 +matplotlib==3.2.1 multitasking==0.0.9 numpy==1.18.1 packaging==20.3 @@ -25,14 +32,19 @@ pandas-datareader==0.8.1 peewee==3.13.3 Pillow==7.1.2 pycparser==2.20 +pycrypto==2.6.1 pydantic==1.4 PyMySQL==0.9.3 pyOpenSSL==19.1.0 pyparsing==2.4.7 python-dateutil==2.8.1 pytz==2020.1 +pyxdg==0.25 PyYAML==5.3.1 requests==2.22.0 +scipy==1.4.1 +seaborn==0.10.1 +SecretStorage==2.3.1 six==1.14.0 SQLAlchemy==1.3.17 starlette==0.12.9 diff --git a/cyberhead/tasker.py b/cyberhead/tasker.py index 3a51f45..3b7b125 100644 --- a/cyberhead/tasker.py +++ b/cyberhead/tasker.py @@ -14,7 +14,7 @@ Celery Initialization If your broker instance is located out of localhost replace replace it below, or set the propper URL if you're lookign to use Redis. ''' -app = Celery('tasker', broker="amqp://localhost//") +app = Celery('tasker', broker="amqp://rabbit//") ''' diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..6a4fe50 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,42 @@ +version: '3' + +services: + cyberhead-mysql: + container_name: cyberhead-mysql + image: mysql + environment: + MYSQL_ROOT_PASSWORD: root + MYSQL_DATABASE: cyberhead + ports: + - "3306:3306" + restart: unless-stopped + cyberhead-rmq: + container_name: cyberhead-rmq + image: 'rabbitmq:3.6-management-alpine' + ports: + - '5672:5672' + - '15672:15672' + restart: unless-stopped + cyberhead: + container_name: cyberhead + environment: + CH_DB_NAME: cyberhead + CH_DB_HOST: database + CH_DB_USER: root + CH_DB_PASSWORD: root + build: + context: ./ + dockerfile: ./Dockerfile + command: sh -c "tail -f /dev/null" + volumes: + - .:/app + depends_on: + - cyberhead-mysql + - cyberhead-rmq + ports: + - 5000:5000 + - 3000:3000 + restart: always + links: + - "cyberhead-rmq:rabbit" + - "cyberhead-mysql:database" diff --git a/docker/Dockerfile b/docker/Dockerfile deleted file mode 100644 index 4c84147..0000000 --- a/docker/Dockerfile +++ /dev/null @@ -1,34 +0,0 @@ -FROM ubuntu:18.04 -LABEL maintainer="CyberHead " \ - org.label-schema.name="CyberHead" \ - org.label-schema.url="http://www.cyberhead.uk/" \ - org.label-schema.version="1.0" - -ENV SERIES 5.0 -ENV LANG C.UTF-8 - -RUN apt-get update \ - && apt-get install apt-utils -y --no-install-recommends \ - python3 \ - python3-pip \ - python3-setuptools \ - git \ - vim - -RUN apt-get install nodejs -y --no-install-recommends \ - npm - -RUN echo "root:root" | chpasswd - -RUN useradd -ms /bin/bash cyberhead -p root -USER cyberhead -WORKDIR /home/cyberhead - - -#RUN cd /home/CyberHead/web \ -# && npm install -y - - -EXPOSE 80 - -#ENTRYPOINT ["/home/CyberHead"] diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml deleted file mode 100644 index cda003f..0000000 --- a/docker/docker-compose.yml +++ /dev/null @@ -1,43 +0,0 @@ -version: '3' - -services: - mysql: - image: mysql - environment: - MYSQL_ROOT_PASSWORD: root - MYSQL_DATABASE: db - ports: - - "3306:3306" - restart: unless-stopped - redis: - image: redis - restart: unless-stopped - rabbitmq: - image: rabbitmq - ports: - - "5672:5672" - restart: unless-stopped - cyberhead: - build: - context: . - image: cyberhead - environment: - CH_PATH: '/home/CyberHead/cyberhead' - CH_DB_NAME: db - CH_DB_HOST: 0.0.0.0 - CH_DB_USER: root - CH_DB_PASSWORD: root - volumes: - - ..:/home/CyberHead - working_dir: /home/CyberHead - - ports: - - "80:80" - depends_on: - - rabbitmq - - mysql - - redis - links: - - mysql:mysql - command: tail -F anything - restart: unless-stopped diff --git a/init.sh b/init.sh new file mode 100755 index 0000000..db7342d --- /dev/null +++ b/init.sh @@ -0,0 +1,2 @@ +python3 ./cyberhead/database.py +python3 ./startup.py diff --git a/startup.py b/startup.py new file mode 100644 index 0000000..67c9694 --- /dev/null +++ b/startup.py @@ -0,0 +1,45 @@ +from cyberhead.database import DataSet +from cyberhead.modules.datasets import yahoo +from sqlalchemy import create_engine +import numpy as np +import pandas +import os + +engine = create_engine(f"mysql+pymysql://{os.getenv('CH_DB_USER')}:{os.getenv('CH_DB_PASSWORD')}@{os.getenv('CH_DB_HOST')}:3306/{os.getenv('CH_DB_NAME')}").connect() + +def allTimeFetch(ticker: str, period: str, interval: str, dataset_id: int): + data = yahoo.download_historical(ticker, period, interval) + data.to_csv('cyberhead/tmp/{}.csv'.format(ticker)) + read_export = pandas.read_csv('cyberhead/tmp/{}.csv'.format(ticker)) + read_export.drop('Adj Close',axis=1, inplace=True) + read_export.columns = ['datetime', 'open_price', 'high_price', 'low_price', 'closing_price', 'volume'] + read_export['dataset_id'] = dataset_id + read_export.to_sql('history', con=engine, if_exists='append', index = False) + return ticker + + +timeseries_ohlc = [ + { + "identifier": "Google 1D", + "reference_symbol":"GOOG1D", + "ticker": "GOOG", + "source": "Yahoo", + "frecuency":"1D" + }, + { + "identifier": "Apple 1D", + "reference_symbol":"AAPL1D", + "ticker": "AAPL", + "source": "Yahoo", + "frecuency":"1D" + } +] + +for data in timeseries_ohlc: + data_set = DataSet.create(identifier=data["identifier"], + reference_symbol=data["reference_symbol"], + symbol=data["ticker"], + source=data["source"], + frecuency=data["frecuency"]) + allTimeFetch(data["ticker"],"max", "1d",data_set.id) + print(f'{data["reference_symbol"]} loaded.') \ No newline at end of file