diff --git a/clickhouse_mysql/__init__.py b/clickhouse_mysql/__init__.py new file mode 100644 index 0000000..d66ec03 --- /dev/null +++ b/clickhouse_mysql/__init__.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from .main import Main + + +def main(): + """Entry point for the application script""" + main = Main() + main.start() + + +if __name__ == '__main__': + main() diff --git a/src/cliopts.py b/clickhouse_mysql/cliopts.py similarity index 99% rename from src/cliopts.py rename to clickhouse_mysql/cliopts.py index 73e9572..1bb4fa2 100644 --- a/src/cliopts.py +++ b/clickhouse_mysql/cliopts.py @@ -2,9 +2,10 @@ # -*- coding: utf-8 -*- import argparse -from .config import Config import logging +from .config import Config + class CLIOpts(object): diff --git a/src/config.py b/clickhouse_mysql/config.py similarity index 100% rename from src/config.py rename to clickhouse_mysql/config.py diff --git a/clickhouse_mysql/converter/__init__.py b/clickhouse_mysql/converter/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/converter/chwriteconverter.py b/clickhouse_mysql/converter/chwriteconverter.py similarity index 100% rename from src/converter/chwriteconverter.py rename to clickhouse_mysql/converter/chwriteconverter.py diff --git a/src/converter/converter.py b/clickhouse_mysql/converter/converter.py similarity index 100% rename from src/converter/converter.py rename to clickhouse_mysql/converter/converter.py diff --git a/src/converter/csvreadconverter.py b/clickhouse_mysql/converter/csvreadconverter.py similarity index 100% rename from src/converter/csvreadconverter.py rename to clickhouse_mysql/converter/csvreadconverter.py diff --git a/src/converter/csvwriteconverter.py b/clickhouse_mysql/converter/csvwriteconverter.py similarity index 100% rename from src/converter/csvwriteconverter.py rename to clickhouse_mysql/converter/csvwriteconverter.py diff --git a/src/daemon.py b/clickhouse_mysql/daemon.py similarity index 100% rename from src/daemon.py rename to clickhouse_mysql/daemon.py diff --git a/clickhouse_mysql/event/__init__.py b/clickhouse_mysql/event/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/event/event.py b/clickhouse_mysql/event/event.py similarity index 100% rename from src/event/event.py rename to clickhouse_mysql/event/event.py diff --git a/clickhouse-mysql b/clickhouse_mysql/main.py old mode 100755 new mode 100644 similarity index 91% rename from clickhouse-mysql rename to clickhouse_mysql/main.py index 9e77d5a..fbed325 --- a/clickhouse-mysql +++ b/clickhouse_mysql/main.py @@ -1,9 +1,9 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from src.cliopts import CLIOpts -from src.pumper import Pumper -from src.daemon import Daemon +from .cliopts import CLIOpts +from .pumper import Pumper +from .daemon import Daemon import sys import multiprocessing as mp @@ -11,9 +11,6 @@ import pprint import json -if sys.version_info[0] < 3: - raise "Must be using Python 3" - class Main(Daemon): diff --git a/src/objectbuilder.py b/clickhouse_mysql/objectbuilder.py similarity index 100% rename from src/objectbuilder.py rename to clickhouse_mysql/objectbuilder.py diff --git a/src/observable.py b/clickhouse_mysql/observable.py similarity index 100% rename from src/observable.py rename to clickhouse_mysql/observable.py diff --git a/clickhouse_mysql/pool/__init__.py b/clickhouse_mysql/pool/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/pool/bbpool.py b/clickhouse_mysql/pool/bbpool.py similarity index 100% rename from src/pool/bbpool.py rename to clickhouse_mysql/pool/bbpool.py index e628ce5..d31ddd1 100644 --- a/src/pool/bbpool.py +++ b/clickhouse_mysql/pool/bbpool.py @@ -2,10 +2,10 @@ # -*- coding: utf-8 -*- import time +import logging from .pool import Pool from ..objectbuilder import ObjectBuilder -import logging # Buckets Belts' Index Generator diff --git a/src/pool/pool.py b/clickhouse_mysql/pool/pool.py similarity index 100% rename from src/pool/pool.py rename to clickhouse_mysql/pool/pool.py diff --git a/src/pumper.py b/clickhouse_mysql/pumper.py similarity index 100% rename from src/pumper.py rename to clickhouse_mysql/pumper.py diff --git a/clickhouse_mysql/reader/__init__.py b/clickhouse_mysql/reader/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/reader/csvreader.py b/clickhouse_mysql/reader/csvreader.py similarity index 99% rename from src/reader/csvreader.py rename to clickhouse_mysql/reader/csvreader.py index 09b53b4..87d1ca7 100644 --- a/src/reader/csvreader.py +++ b/clickhouse_mysql/reader/csvreader.py @@ -1,11 +1,12 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- +import csv +import os + from .reader import Reader from ..event.event import Event from ..converter.csvreadconverter import CSVReadConverter -import csv -import os class CSVReader(Reader): diff --git a/src/reader/mysqlreader.py b/clickhouse_mysql/reader/mysqlreader.py similarity index 99% rename from src/reader/mysqlreader.py rename to clickhouse_mysql/reader/mysqlreader.py index 4f2ff4d..a5d371b 100644 --- a/src/reader/mysqlreader.py +++ b/clickhouse_mysql/reader/mysqlreader.py @@ -1,13 +1,16 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from .reader import Reader -from ..event.event import Event +import time +import logging + from pymysqlreplication import BinLogStreamReader from pymysqlreplication.row_event import WriteRowsEvent, UpdateRowsEvent, DeleteRowsEvent + +from .reader import Reader +from ..event.event import Event #from pymysqlreplication.event import QueryEvent, RotateEvent, FormatDescriptionEvent -import time -import logging + class MySQLReader(Reader): diff --git a/src/reader/reader.py b/clickhouse_mysql/reader/reader.py similarity index 100% rename from src/reader/reader.py rename to clickhouse_mysql/reader/reader.py diff --git a/src/tablebuilder.py b/clickhouse_mysql/tablebuilder.py similarity index 100% rename from src/tablebuilder.py rename to clickhouse_mysql/tablebuilder.py diff --git a/src/tablemigrator.py b/clickhouse_mysql/tablemigrator.py similarity index 100% rename from src/tablemigrator.py rename to clickhouse_mysql/tablemigrator.py diff --git a/src/tableprocessor.py b/clickhouse_mysql/tableprocessor.py similarity index 100% rename from src/tableprocessor.py rename to clickhouse_mysql/tableprocessor.py diff --git a/clickhouse_mysql/writer/__init__.py b/clickhouse_mysql/writer/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/writer/chcsvwriter.py b/clickhouse_mysql/writer/chcsvwriter.py similarity index 99% rename from src/writer/chcsvwriter.py rename to clickhouse_mysql/writer/chcsvwriter.py index 99a9054..1d551f9 100644 --- a/src/writer/chcsvwriter.py +++ b/clickhouse_mysql/writer/chcsvwriter.py @@ -1,12 +1,13 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from .writer import Writer - import os import time import logging +from .writer import Writer + + class CHCSVWriter(Writer): host = None diff --git a/src/writer/chwriter.py b/clickhouse_mysql/writer/chwriter.py similarity index 99% rename from src/writer/chwriter.py rename to clickhouse_mysql/writer/chwriter.py index 0e54d4c..b46f72d 100644 --- a/src/writer/chwriter.py +++ b/clickhouse_mysql/writer/chwriter.py @@ -1,11 +1,13 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- +import logging +import sys + from clickhouse_driver.client import Client + from .writer import Writer from ..event.event import Event -import logging -import sys class CHWriter(Writer): diff --git a/src/writer/csvwriter.py b/clickhouse_mysql/writer/csvwriter.py similarity index 99% rename from src/writer/csvwriter.py rename to clickhouse_mysql/writer/csvwriter.py index 2074683..8b5cea0 100644 --- a/src/writer/csvwriter.py +++ b/clickhouse_mysql/writer/csvwriter.py @@ -1,12 +1,13 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from .writer import Writer -from ..event.event import Event import csv import os.path import logging +from .writer import Writer +from ..event.event import Event + class CSVWriter(Writer): diff --git a/src/writer/poolwriter.py b/clickhouse_mysql/writer/poolwriter.py similarity index 99% rename from src/writer/poolwriter.py rename to clickhouse_mysql/writer/poolwriter.py index be6a662..bf07a4e 100644 --- a/src/writer/poolwriter.py +++ b/clickhouse_mysql/writer/poolwriter.py @@ -1,10 +1,11 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- +import logging + from .writer import Writer from ..event.event import Event from ..pool.bbpool import BBPool -import logging class PoolWriter(Writer): diff --git a/src/writer/processwriter.py b/clickhouse_mysql/writer/processwriter.py similarity index 99% rename from src/writer/processwriter.py rename to clickhouse_mysql/writer/processwriter.py index 6f67df6..a08b377 100644 --- a/src/writer/processwriter.py +++ b/clickhouse_mysql/writer/processwriter.py @@ -1,10 +1,11 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from .writer import Writer import multiprocessing as mp import logging +from .writer import Writer + class ProcessWriter(Writer): diff --git a/src/writer/writer.py b/clickhouse_mysql/writer/writer.py similarity index 100% rename from src/writer/writer.py rename to clickhouse_mysql/writer/writer.py diff --git a/setup.py b/setup.py index 3d4d469..6c90a3f 100644 --- a/setup.py +++ b/setup.py @@ -1,15 +1,68 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -version = "0.1" +from setuptools import setup, find_packages setup( - name="clickhouse-mysql-data-reader", - version=version, + name="clickhouse-mysql", + + # version should comply with PEP440 + version='0.0.1', + + description='ClickHouse Data Reader', + long_description='ClickHouse Data Reader', + + # homepage url="https://github.com/altinity/clickhouse-mysql-data-reader", - author="sunsingerus", + + author="Vladislav Klimenko", author_email="sunsingerus@gmail.com", - description=("ClickHouse data reader"), + license="MIT", - install_requires=['pymysqlreplication'], + + # see https://pypi.python.org/pypi?%3Aaction=list_classifiers + classifiers=[ + # How mature is this project? Common values are + # 3 - Alpha + # 4 - Beta + # 5 - Production/Stable + 'Development Status :: 3 - Alpha', + + 'Intended Audience :: Developers', + 'Topic :: Database', + + # should match license above + 'License :: OSI Approved :: MIT License', + + # supported Python versions + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + ], + + # what does the project relate to? + keywords='clickhouse mysql data migration', + + # list of packages to be included into project + packages=find_packages(exclude=['contrib', 'docs', 'tests']), + + # run-time dependencies + # these will be installed by pip + # https://packaging.python.org/en/latest/requirements.html + install_requires=[ + 'mysqlclient', + 'mysql-replication', + 'clickhouse-driver', + ], + + # cross-platform support for pip to create the appropriate form of executable + entry_points={ + 'console_scripts': [ + 'clickhouse-mysql=clickhouse_mysql:main', + ], + }, + +# python_requires='>=3.3', )