From 1108dc1f5dad064f025d9fd93efa0d4783a87eff Mon Sep 17 00:00:00 2001 From: leeyi Date: Tue, 24 Sep 2019 23:08:41 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BD=BF=E7=94=A8=20pyyaml=EF=BC=8C=E9=85=8D?= =?UTF-8?q?=E7=BD=AE=E6=96=87=E4=BB=B6=E3=80=81=E4=BB=A3=E7=A0=81=E5=81=9A?= =?UTF-8?q?=E7=9B=B8=E5=BA=94=E8=B0=83=E6=95=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: leeyi --- Pipfile | 2 +- README.md | 21 +-- README_zh.md | 21 +-- demo_dot.env | 3 +- setup.py | 2 +- tests/app_demo/configs/dev.ini | 98 ----------- tests/app_demo/configs/dev.yaml | 220 ++++++++++++++++++++++++ trest/amqp/__init__.py | 2 +- trest/cache/__init__.py | 13 +- trest/cache/backends/__init__.py | 4 +- trest/cache/backends/base.py | 9 +- trest/cache/backends/dummy.py | 6 +- trest/cache/backends/localcache.py | 9 +- trest/cache/backends/memcached.py | 14 +- trest/cache/backends/rediscache.py | 30 +--- trest/config.py | 257 ----------------------------- trest/config/__init__.py | 71 ++++++++ trest/config/config.py | 226 +++++++++++++++++++++++++ trest/db/dbalchemy.py | 22 +-- trest/middleware/dbalchemy.py | 23 +-- trest/middleware/response.py | 2 +- trest/storage.py | 25 ++- trest/utils/encrypter.py | 4 +- trest/utils/utime.py | 12 +- trest/webserver.py | 12 +- 25 files changed, 633 insertions(+), 475 deletions(-) delete mode 100644 tests/app_demo/configs/dev.ini create mode 100644 tests/app_demo/configs/dev.yaml delete mode 100644 trest/config.py create mode 100644 trest/config/__init__.py create mode 100644 trest/config/config.py diff --git a/Pipfile b/Pipfile index 5bc6be9..c60198a 100644 --- a/Pipfile +++ b/Pipfile @@ -9,9 +9,9 @@ gino = "*" [packages] tornado = "*" +pyyaml = "*" mysqlclient = "*" sqlalchemy = "*" -sqlalchemy-utils = "*" redis = "==2.10.6" rsa = "*" pycryptodome = "*" diff --git a/README.md b/README.md index d1886bd..7ab3e3c 100644 --- a/README.md +++ b/README.md @@ -46,8 +46,8 @@ tree -I '*svn|*node_module*|*git|py3|*.pyc|__pycache__|statics' │   └── app2 │   └── app3 ├── configs -│   ├── dev.ini -│   └── local.ini +│   ├── dev.yaml +│   └── local.yaml ├── datas │   ├── locales │   │   ├── en_US.csv @@ -70,10 +70,10 @@ tree -I '*svn|*node_module*|*git|py3|*.pyc|__pycache__|statics' * .env 环境配置文件，只有一个section [sys]，一个变量 TREST_ENV * configs 应用配置文件 - * configs/local.ini 本地开发环境相关配置 - * configs/dev.ini 开发环境相关配置 - * configs/test.ini 测试环境相关配置 - * configs/product.ini 生产环境相关配置 + * configs/local.yaml 本地开发环境相关配置 + * configs/dev.yaml 开发环境相关配置 + * configs/test.yaml 测试环境相关配置 + * configs/product.yaml 生产环境相关配置 * applications 应用rest api相关代码 * applications/common/models 公共应用数据模型层 * applications/common/services 公共应用服务层 @@ -148,8 +148,8 @@ if __name__ == "__main__": 在 项目根目录（ ROOT_PATH ） 下面创建 [.env 文件](https://gitee.com/leeyi/trest/blob/master/demo_dot.env) ``` # TREST_ENV is not one of the local, dev, test, or product -[sys] -TREST_ENV = dev +TREST_ENV : dev + ``` run @@ -157,11 +157,12 @@ run pipenv install --skip-lock pipenv shell python server.py --port=5080 +python tests/app_demo/server.py --port=5081 ``` -f'{ROOT_PATH}/configs/{env}.ini' demo +f'{ROOT_PATH}/configs/{env}.yaml' demo -like this [./tests/app_demo/configs/dev.ini](https://gitee.com/leeyi/trest/blob/master/tests/app_demo/configs/dev.ini) +like this [./tests/app_demo/configs/dev.yaml](https://gitee.com/leeyi/trest/blob/master/tests/app_demo/configs/dev.yaml) # [开发约定](https://gitee.com/leeyi/trest/blob/master/promise.md) diff --git a/README_zh.md b/README_zh.md index d1886bd..7ab3e3c 100644 --- a/README_zh.md +++ b/README_zh.md @@ -46,8 +46,8 @@ tree -I '*svn|*node_module*|*git|py3|*.pyc|__pycache__|statics' │   └── app2 │   └── app3 ├── configs -│   ├── dev.ini -│   └── local.ini +│   ├── dev.yaml +│   └── local.yaml ├── datas │   ├── locales │   │   ├── en_US.csv @@ -70,10 +70,10 @@ tree -I '*svn|*node_module*|*git|py3|*.pyc|__pycache__|statics' * .env 环境配置文件，只有一个section [sys]，一个变量 TREST_ENV * configs 应用配置文件 - * configs/local.ini 本地开发环境相关配置 - * configs/dev.ini 开发环境相关配置 - * configs/test.ini 测试环境相关配置 - * configs/product.ini 生产环境相关配置 + * configs/local.yaml 本地开发环境相关配置 + * configs/dev.yaml 开发环境相关配置 + * configs/test.yaml 测试环境相关配置 + * configs/product.yaml 生产环境相关配置 * applications 应用rest api相关代码 * applications/common/models 公共应用数据模型层 * applications/common/services 公共应用服务层 @@ -148,8 +148,8 @@ if __name__ == "__main__": 在 项目根目录（ ROOT_PATH ） 下面创建 [.env 文件](https://gitee.com/leeyi/trest/blob/master/demo_dot.env) ``` # TREST_ENV is not one of the local, dev, test, or product -[sys] -TREST_ENV = dev +TREST_ENV : dev + ``` run @@ -157,11 +157,12 @@ run pipenv install --skip-lock pipenv shell python server.py --port=5080 +python tests/app_demo/server.py --port=5081 ``` -f'{ROOT_PATH}/configs/{env}.ini' demo +f'{ROOT_PATH}/configs/{env}.yaml' demo -like this [./tests/app_demo/configs/dev.ini](https://gitee.com/leeyi/trest/blob/master/tests/app_demo/configs/dev.ini) +like this [./tests/app_demo/configs/dev.yaml](https://gitee.com/leeyi/trest/blob/master/tests/app_demo/configs/dev.yaml) # [开发约定](https://gitee.com/leeyi/trest/blob/master/promise.md) diff --git a/demo_dot.env b/demo_dot.env index c0e393e..0f29899 100644 --- a/demo_dot.env +++ b/demo_dot.env @@ -1,3 +1,2 @@ # TREST_ENV is not one of the local, dev, test, or product -[sys] -TREST_ENV = dev +TREST_ENV : dev diff --git a/setup.py b/setup.py index f817d44..3d22234 100644 --- a/setup.py +++ b/setup.py @@ -22,9 +22,9 @@ platforms='any', install_requires=[ 'tornado>=6.0.0', + 'pyyaml', 'mysqlclient', 'sqlalchemy', - 'sqlalchemy-utils', 'redis==2.10.6', 'rsa', 'pycryptodome', diff --git a/tests/app_demo/configs/dev.ini b/tests/app_demo/configs/dev.ini deleted file mode 100644 index 9ce0cd6..0000000 --- a/tests/app_demo/configs/dev.ini +++ /dev/null @@ -1,98 +0,0 @@ -# f'{ROOT_PATH}/configs/{env}.ini' -# 注意配置解析出来都是字符串，请不要带单引号或者双引号 -# 例如 '0.0.0.0' "0.0.0.0" 都会报错 - -[sys] -arbitrary_ip = 0.0.0.0 -port = 5080 -local_ip = 127.0.0.1 -translation = true -time_zone = Asia/Shanghai -language_code = zh-hans -installed_app = app1 -login_pwd_rsa_encrypt = True -default_aes_secret = 883d65f06fd447f3a1e69a36e73f58e0 -admin_session_key = de0b3fb0c2f44563944a8cccca7f225a -front_session_key = 171630947de24c969c28b2d178c4e0fe -valid_code_key = ab1195c6f0084b4f8b007d3aa7628a38 -token_key = f30a2331813f46d0adc2bcf26fcbbbf4 -rabbitmq_config = -sentry_url = -config_cache_prefix = conf: -user_cache_prefix = user: -admin_cache_prefix = admin: - -# 超级管理员角色ID -super_role_id = 1 -default_role_id = 2 - -[tornado] -debug = true -xsrf_cookies = true -xheaders = true -cookie_secret = e921bfcd-ace4-4124-8657-c57a162365f6 - -[session] -cache_alias = default_redis -name = nkzpg9NKBpKS2iaK -cookie_domain = -cookie_path = / -expires = 86400 -secret = fLjUfxqXtfNoIldA0A0J -version = v0.1.0 - -[sqlalchemy] -# (s秒) -ping_db = 300 -# 每次取出ping多少个连接 -ping_conn_count = 5 -connect_timeout = 3 -echo = true -echo_pool = true -max_overflow = 10 -pool_timeout = 5 -encoding = utf8 -pool_size = 5 -pool_recycle = 3600 -poolclass = QueuePool - -[default_redis] -location = 127.0.0.1:6379 -db = 0 -password = abc123456 -ping_interval = 120 -parser_class = redis.connection.DefaultParser -socket_timeout = 2 -socket_connect_timeout = 2 - -[redis] -host = 127.0.0.1 -port = 6379 -password = abc123456 -charset = utf8 -db = 3 - -[db_master] -driver = mysql+mysqldb -user = root -password = 123456 -host = 127.0.0.1 -port = 3306 -database = db_py_admin -charset = utf8mb4 - -[db_slave] -driver = mysql+mysqldb -user = root -password = 123456 -host = 127.0.0.1 -port = 3306 -database = db_py_admin -charset = utf8mb4 - -[email] -from_name = -from_addr = -smtp_sever = -smtp_port = -auth_code = diff --git a/tests/app_demo/configs/dev.yaml b/tests/app_demo/configs/dev.yaml new file mode 100644 index 0000000..efade25 --- /dev/null +++ b/tests/app_demo/configs/dev.yaml @@ -0,0 +1,220 @@ +# f'{ROOT_PATH}/configs/{env}.yaml' +# 注意配置解析出来都是字符串，请不要带单引号或者双引号 +# 例如 '0.0.0.0' "0.0.0.0" 都会报错 + + +debug : true +xsrf_cookies : true +xheaders : true + +arbitrary_ip : 0.0.0.0 +port : 5080 +local_ip : 127.0.0.1 +translation : true +time_zone : Asia/Shanghai +language_code : zh-hans +cookie_secret : e921bfcd-ace4-4124-8657-c57a162365f6 +login_pwd_rsa_encrypt : True +default_aes_secret : 883d65f06fd447f3a1e69a36e73f58e0 +admin_session_key : de0b3fb0c2f44563944a8cccca7f225a +front_session_key : 171630947de24c969c28b2d178c4e0fe +valid_code_key : ab1195c6f0084b4f8b007d3aa7628a38 +token_key : f30a2331813f46d0adc2bcf26fcbbbf4 + + +INSTALLED_APPS : + - app1 + +rabbitmq_config : '' +sentry_url : +config_cache_prefix : 'conf:' +user_cache_prefix : 'user:' +admin_cache_prefix : 'admin:' + +# 超级管理员角色ID +SUPER_ROLE_ID : 1 +DEFAULT_ROLE_ID : 2 + +# Super Admin 必须是 int 型数据 +SUPER_ADMIN : + - 1 # admin uid + - 2 # + +# 是否开启国际化 +translation : true + +PASSWORD_HASHERS : + # 第一个元素为默认加密方式 + - 'trest.utils.hasher.PBKDF2PasswordHasher' + - 'trest.utils.hasher.PBKDF2SHA1PasswordHasher' + +# 中间件 # +# ########### +MIDDLEWARE_CLASSES : + - 'trest.middleware.dbalchemy.DBAlchemyMiddleware' + - 'trest.middleware.AccessLogMiddleware' + - 'trest.middleware.PushToMQMiddleware' + +session : + cache_alias : default_redis + name : nkzpg9NKBpKS2iaK + cookie_domain : + cookie_path : / + expires : 86400 + secret : fLjUfxqXtfNoIldA0A0J + version : v0.1.0 + +# sqlalchemy配置，列出部分，可自行根据sqlalchemy文档增加配置项 +# 该配置项对所有连接全局共享 +sqlalchemy : + # (s秒) + ping_db : 300 + # 每次取出ping多少个连接 + ping_conn_count : 5 + 'sqlalchemy.connect_args' : { + 'connect_timeout' : 3 + } + 'sqlalchemy.echo' : true + 'sqlalchemy.max_overflow' : 10 + 'sqlalchemy.echo_pool' : true + 'sqlalchemy.pool_timeout' : 5 + 'sqlalchemy.encoding' : utf8 + 'sqlalchemy.pool_size' : 5 + 'sqlalchemy.pool_recycle' : 3600 + # 手动指定连接池类 + 'sqlalchemy.poolclass' : QueuePool + +# 数据库连接字符串，元祖， +# 每组为n个数据库连接，有且只有一个master，可配与不配slave +DATABASE_CONNECTION : + default : + connections : + - + ROLE: 'master' + DRIVER : 'mysql+mysqldb' + UID : root + # 进过AES加密的密码，格式 aes::: + ciphertext + PASSWD : '123456' + HOST : '127.0.0.1' + PORT : 3306 + DATABASE : 'db_py_admin' + QUERY : {'charset' : 'utf8mb4'} + - + ROLE : 'slave' + DRIVER : 'mysql+mysqldb' + UID : root + # 进过AES加密的密码，格式 aes::: + ciphertext + PASSWD : '123456' + HOST : '127.0.0.1' + PORT : 3306 + DATABASE : 'db_py_admin' + QUERY : {'charset' : 'utf8mb4'} + +########### +# 缓存配置 # +########### +CACHES : + 'default': + 'BACKEND': 'trest.cache.backends.localcache.LocMemCache' + 'LOCATION': 'process_cache' + 'OPTIONS': + 'MAX_ENTRIES': 10000 + 'CULL_FREQUENCY': 3 + + 'default_redis': + 'BACKEND': 'trest.cache.backends.rediscache.RedisCache' + 'LOCATION': '127.0.0.1:6379' + 'OPTIONS': + 'DB': 0 + 'PASSWORD': '' + 'PARSER_CLASS': 'redis.connection.DefaultParser' + # 定时ping redis连接池，防止被服务端断开连接（s秒） + 'PING_INTERVAL': 120 + 'POOL_KWARGS': + 'socket_timeout': 2 + 'socket_connect_timeout': 2 + +# 配置模版引擎 +# 引入相应的TemplateLoader即可 +# 若使用自带的请留空 +# 支持mako和jinja2 +# mako设置为 tornado.template.mako_loader.MakoTemplateLoader +# jinj2设置为 tornado.template.jinja2_loader.Jinja2TemplateLoader +# 初始化参数请参照jinja的Environment或mako的TemplateLookup,不再详细给出 +TEMPLATE_CFG : + 'template_engine': '' + # 模版路径由tornado.handler中commonhandler重写， + # 无需指定，模版将存在于每个应用的根目录下 + #通用选项 + 'filesystem_checks': True + 'cache_directory': '../_tmpl_cache' # 模版编译文件目录,通用选项 + # 暂存入内存的模版项，可以提高性能，mako选项,详情见mako文档 + 'collection_size': 50 + # 类似于mako的collection_size，设定为-1为不清理缓存，0则每次都会重编译模板 + 'cache_size': 0 + # 格式化异常输出，mako专用 + 'format_exceptions': False + # 默认转义设定，jinja2专用 + 'autoescape': False + +# tornado日志功能配置 +# Logging中有 +# NOTSET < DEBUG < INFO < WARNING < ERROR < CRITICAL 这几种级别， +# 日志会记录设置级别以上的日志 +# when 时间 按照哪种时间单位滚动（可选s-按秒，m-按分钟，h-按小时，d-按天，w0-w6-按指定的星期几，midnight-在午夜） +log_cfg: + log_dir: '' + standard_format : &standard_format '[%(asctime)s][%(threadName)s:%(thread)d][task_id:%(name)s][%(filename)s:%(lineno)d] [%(levelname)s][%(message)s]' + + logging : + - + 'name': 'access_log' + 'log_to_stderr': True + 'filename': 'access_log.log' + - + 'name': 'tornado.debug.log' + 'level': 'DEBUG' + 'log_to_stderr': True + 'when': 'w0' + 'interval': 1 + 'formatter': *standard_format + 'filename': 'debug_log.log' + - + 'name': 'tornado.info.log' + 'level': 'INFO' + 'log_to_stderr': True + 'when': 'midnight' + 'interval': 1 + 'formatter': *standard_format + 'filename': 'info_log.log' + - + 'name': 'tornado.warning.log' + 'level': 'WARNING' + 'log_to_stderr': True + 'when': 'midnight' + 'interval': 1 + 'formatter': *standard_format + 'filename': 'warning_log.log' + - + 'name': 'tornado.error.log' + 'level': 'ERROR' + 'log_to_stderr': True + 'when': 'midnight' + 'interval': 1 + 'formatter': *standard_format + 'filename': 'error_log.log' + - + 'name': 'tornado.critical.log' + 'level': 'CRITICAL' + 'log_to_stderr': True + 'when': 'midnight' + 'interval': 1 + 'formatter': *standard_format + 'filename': 'critical_log.log' + +email : + from_name : + from_addr : + smtp_sever : + smtp_port : + auth_code : diff --git a/trest/amqp/__init__.py b/trest/amqp/__init__.py index 312eb3c..50268d5 100644 --- a/trest/amqp/__init__.py +++ b/trest/amqp/__init__.py @@ -54,7 +54,7 @@ def push_to_mq(param, option): msg['msg_type'] = option.get('msg_type', '') msg['msg_md5'] = func.md5(json.dumps(param)) msg['msg'] = param - pusher = Publisher(settings.sys.rabbitmq_config) + pusher = Publisher(settings.rabbitmq_config) option['exchange_type'] = option.get('exchange_type', 'topic') res = pusher.push(msg, option) # print(res) diff --git a/trest/cache/__init__.py b/trest/cache/__init__.py index 9a0f578..1549e49 100644 --- a/trest/cache/__init__.py +++ b/trest/cache/__init__.py @@ -9,9 +9,9 @@ from trest.config import settings from ..exception import ConfigError -from .backends.base import InvalidCacheBackendError -from .backends.base import CacheKeyWarning from .backends.base import BaseCache +from .backends.base import CacheKeyWarning +from .backends.base import InvalidCacheBackendError BACKENDS = { @@ -29,7 +29,6 @@ if DEFAULT_CACHE_ALIAS not in settings.CACHES: raise ConfigError("You must define a '%s' cache" % DEFAULT_CACHE_ALIAS) - def _create_cache(backend, **kwargs): try: # Try to get the CACHES entry for the given backend name first @@ -55,14 +54,11 @@ def _create_cache(backend, **kwargs): "Could not find backend '%s': %s" % (backend, e)) return backend_cls(location, params) - class CacheHandler(object): """ A Cache Handler to manage access to Cache instances. - Ensures only one instance of each alias exists per thread. """ - def __init__(self): self._caches = local() @@ -91,14 +87,11 @@ def __getitem__(self, alias): self._caches.caches[alias] = cache return cache - def all(self): return getattr(self._caches, 'caches', {}).values() - caches = CacheHandler() - class DefaultCacheProxy(object): """ Proxy access to the default Cache object's attributes. @@ -125,10 +118,8 @@ def __eq__(self, other): def __ne__(self, other): return caches[DEFAULT_CACHE_ALIAS] != other - cache = DefaultCacheProxy() - def close_caches(**kwargs): # Some caches -- python-memcached in particular -- need to do a cleanup at the # end of a request cycle. If not implemented in a particular backend diff --git a/trest/cache/backends/__init__.py b/trest/cache/backends/__init__.py index 98c2df8..2fe13e7 100644 --- a/trest/cache/backends/__init__.py +++ b/trest/cache/backends/__init__.py @@ -1,3 +1,3 @@ """ -torngas cache backends -""" \ No newline at end of file +trest cache backends +""" diff --git a/trest/cache/backends/base.py b/trest/cache/backends/base.py index 76ee46c..1203fe8 100644 --- a/trest/cache/backends/base.py +++ b/trest/cache/backends/base.py @@ -4,25 +4,22 @@ from __future__ import unicode_literals import time + from tornado.util import import_object from trest.exception import ConfigError + class InvalidCacheBackendError(ConfigError): pass - class CacheKeyWarning(RuntimeWarning): pass - # Stub class to ensure not passing in a `timeout` argument results in # the default timeout DEFAULT_TIMEOUT = object() - - - def default_key_func(key, key_prefix, version): """ Default function to generate keys. @@ -237,4 +234,4 @@ def decr_version(self, key, delta=1, version=None): class BaseCache(CacheMixin, CacheClient): - pass \ No newline at end of file + pass diff --git a/trest/cache/backends/dummy.py b/trest/cache/backends/dummy.py index 61de40c..2844728 100644 --- a/trest/cache/backends/dummy.py +++ b/trest/cache/backends/dummy.py @@ -1,4 +1,8 @@ -"Dummy cache backend" +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" + Dummy cache backend" +""" from .base import BaseCache, DEFAULT_TIMEOUT diff --git a/trest/cache/backends/localcache.py b/trest/cache/backends/localcache.py index 6aca54a..9e7a2c2 100644 --- a/trest/cache/backends/localcache.py +++ b/trest/cache/backends/localcache.py @@ -1,7 +1,12 @@ -"Thread-safe in-memory cache backend." +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" + Thread-safe in-memory cache backend. +""" import time -from .base import BaseCache, DEFAULT_TIMEOUT +from .base import BaseCache +from .base import DEFAULT_TIMEOUT from trest.utils import RWLock diff --git a/trest/cache/backends/memcached.py b/trest/cache/backends/memcached.py index ffa6872..c7c5a69 100644 --- a/trest/cache/backends/memcached.py +++ b/trest/cache/backends/memcached.py @@ -1,17 +1,23 @@ -"Memcached cache backend" +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" + Memcached cache backend +""" import time import pickle import warnings -from .base import CacheKeyWarning -from .base import BaseCache, DEFAULT_TIMEOUT from trest.utils import string_types from trest.utils.func import safestr from trest.utils import cached_property +from .base import CacheKeyWarning +from .base import BaseCache +from .base import DEFAULT_TIMEOUT + + # Memcached does not accept keys longer than this. MEMCACHE_MAX_KEY_LENGTH = 250 - class BaseMemcachedCache(BaseCache): def __init__(self, server, params, library, value_not_found_exception): super(BaseMemcachedCache, self).__init__(params) diff --git a/trest/cache/backends/rediscache.py b/trest/cache/backends/rediscache.py index 573a395..acc23d5 100644 --- a/trest/cache/backends/rediscache.py +++ b/trest/cache/backends/rediscache.py @@ -1,12 +1,16 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- import sys -from .base import CacheMixin, CacheClient, InvalidCacheBackendError, DEFAULT_TIMEOUT from tornado.util import import_object from trest.storage import SortedDict from trest.utils.func import safestr from trest.exception import ConfigError +from .base import CacheMixin +from .base import CacheClient +from .base import InvalidCacheBackendError +from .base import DEFAULT_TIMEOUT + try: import cPickle as pickle except ImportError: @@ -20,29 +24,7 @@ from redis.connection import UnixDomainSocketConnection, Connection from redis.connection import DefaultParser -PY3 = (sys.version_info >= (3,)) - -if PY3: - bytes_type = bytes -else: - bytes_type = str - - -def python_2_unicode_compatible(klass): - """ - A decorator that defines __unicode__ and __str__ methods under Python 2. - Under Python 3 it does nothing. - - To support Python 2 and 3 with a single code base, define a __str__ method - returning text and apply this decorator to the class. - """ - if not PY3: - klass.__unicode__ = klass.__str__ - klass.__str__ = lambda self: self.__unicode__().encode('utf-8') - return klass - -@python_2_unicode_compatible class CacheKey(object): """ A stub string class that we can use to check if a key was created already. @@ -348,7 +330,7 @@ def get_many(self, keys, version=None): if value is None: continue value = self.unpickle(value) - if isinstance(value, bytes_type): + if isinstance(value, bytes): value = safestr(value) recovered_data[map_keys[key]] = value return recovered_data diff --git a/trest/config.py b/trest/config.py deleted file mode 100644 index a80f0ba..0000000 --- a/trest/config.py +++ /dev/null @@ -1,257 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import os -import configparser -from tornado.options import options -from .storage import storage -from .exception import ConfigError - - -# 检查全局变量 ROOT_PATH 设置 -if hasattr(options, 'ROOT_PATH') and os.path.exists(options.ROOT_PATH): - ROOT_PATH = options.ROOT_PATH -else: - raise ConfigError('ROOT_PATH is not configured') - -dcfg = configparser.ConfigParser() -dcfg.read(f'{ROOT_PATH}/.env', encoding='utf8') - -env = dcfg.get('sys','TREST_ENV') -# 检查系统环境变量 TREST_ENV 设置 -if env not in ['local', 'dev', 'test', 'product']: - msg = f'The system variable TREST_ENV ({env}) is not one of the local, dev, test, or product' - raise ConfigError(msg) - -class ConfigParser(configparser.ConfigParser): - def as_dict(self): - """ - 将 configparser.ConfigParser().read() 读到的数据转换为 storage - """ - d = storage(self._sections) - for k in d: - d[k] = storage(d[k]) - return d - -cfg = ConfigParser() -_ini = f'{ROOT_PATH}/configs/{env}.ini' -if not(os.path.isfile(_ini) and os.access(_ini, os.R_OK)): - raise ConfigError(f'The ENV file({_ini}) does not exist or is unreadable') -cfg.read(_ini, encoding='utf8') - -settings = cfg.as_dict() -settings.ROOT_PATH = ROOT_PATH -settings.STATIC_PATH = os.path.join(ROOT_PATH, 'statics') -settings.ENV = env -settings.debug = cfg.getboolean('tornado', 'debug') -settings.xheaders = cfg.getboolean('tornado', 'xheaders') -settings.xsrf_cookies = cfg.getboolean('tornado', 'xsrf_cookies') - -_app = cfg.get('sys', 'installed_app') -settings.INSTALLED_APPS = [n.strip() for n in _app.split(',') if n] - -# Super Admin 必须是 int 型数据 -settings.SUPER_ADMIN = [ - 1, # admin uid - 2, # -] -# 超级管理员角色ID -settings.SUPER_ROLE_ID = 1 -settings.DEFAULT_ROLE_ID = 2 -# 系统角色，非超级管理员不允许编辑权限和删除 -settings.SYS_ROLE = [ - settings.SUPER_ROLE_ID, - settings.DEFAULT_ROLE_ID -] - -# 是否开启国际化 -settings.translation = cfg.getboolean('sys', 'translation') -settings.TRANSLATIONS_CONF = storage({ - 'translations_dir': os.path.join(ROOT_PATH, 'datas/locales'), - 'locale_default': 'zh_CN', - 'use_accept_language': True -}) - -# tornado全局配置 -settings.TORNADO_CONF = { - 'xsrf_cookies': settings.xsrf_cookies, - 'login_url': '/admin/login', - 'cookie_secret': cfg.get('tornado', 'cookie_secret'), - # 'ui_modules': ui_modules, - 'template_path': os.path.join(ROOT_PATH, 'applications/admin/templates'), - 'static_path': settings.STATIC_PATH, - # 安全起见，可以定期生成新的cookie 秘钥，生成方法： - # base64.b64encode(uuid.uuid4().bytes + uuid.uuid4().bytes) -} - -# 中间件 # -# ########### -settings.MIDDLEWARE_CLASSES = ( - 'trest.middleware.dbalchemy.DBAlchemyMiddleware', - 'trest.middleware.AccessLogMiddleware', - 'trest.middleware.PushToMQMiddleware', -) - -# 数据库连接字符串， -# 元祖，每组为n个数据库连接，有且只有一个master，可配与不配slave -settings.DATABASE_CONNECTION = { - 'default': { - 'connections': [ - { - 'ROLE': 'master', - 'DRIVER': cfg.get('db_master', 'driver'), - 'UID': cfg.get('db_master', 'user'), - # 进过AES加密的密码，格式 aes::: + ciphertext - 'PASSWD': cfg.get('db_master', 'password'), - 'HOST': cfg.get('db_master', 'host'), - 'PORT': cfg.getint('db_master', 'port'), - 'DATABASE': cfg.get('db_master', 'database'), - 'QUERY': {'charset': cfg.get('db_master', 'charset')} - }, - { - 'ROLE': 'slave', - 'DRIVER': cfg.get('db_slave', 'driver'), - 'UID': cfg.get('db_slave', 'user'), - # 进过AES加密的密码，格式 aes::: + ciphertext - 'PASSWD': cfg.get('db_slave', 'password'), - 'HOST': cfg.get('db_slave', 'host'), - 'PORT': cfg.getint('db_slave', 'port'), - 'DATABASE': cfg.get('db_slave', 'database'), - 'QUERY': {'charset': cfg.get('db_slave', 'charset')} - } - ] - } -} - -# sqlalchemy配置，列出部分，可自行根据sqlalchemy文档增加配置项 -# 该配置项对所有连接全局共享 -settings.SQLALCHEMY_CONFIGURATION = { - 'sqlalchemy.connect_args': { - 'connect_timeout': cfg.getint('sqlalchemy', 'connect_timeout') - }, - 'sqlalchemy.echo': cfg.getboolean('sqlalchemy', 'echo'), - 'sqlalchemy.max_overflow': cfg.getint('sqlalchemy', 'max_overflow'), - 'sqlalchemy.echo_pool': cfg.getboolean('sqlalchemy', 'echo_pool'), - 'sqlalchemy.pool_timeout': cfg.getint('sqlalchemy', 'pool_timeout'), - 'sqlalchemy.encoding': cfg.get('sqlalchemy', 'encoding'), - 'sqlalchemy.pool_size': cfg.getint('sqlalchemy', 'pool_size'), - 'sqlalchemy.pool_recycle': cfg.getint('sqlalchemy', 'pool_recycle'), - # 手动指定连接池类 - 'sqlalchemy.poolclass': cfg.get('sqlalchemy', 'poolclass'), -} - -########### -# 缓存配置 # -########### -settings.CACHES = { - 'default': { - 'BACKEND': 'trest.cache.backends.localcache.LocMemCache', - 'LOCATION': 'process_cache', - 'OPTIONS': { - 'MAX_ENTRIES': 10000, - 'CULL_FREQUENCY': 3 - } - }, - 'default_redis': { - 'BACKEND': 'trest.cache.backends.rediscache.RedisCache', - 'LOCATION': cfg.get('default_redis', 'location'), - 'OPTIONS': { - 'DB': cfg.getint('default_redis', 'db'), - 'PASSWORD': cfg.get('default_redis', 'password'), - 'PARSER_CLASS': cfg.get('default_redis', 'parser_class'), - 'POOL_KWARGS': { - 'socket_timeout': cfg.getint('default_redis', 'socket_timeout'), - 'socket_connect_timeout': cfg.getint('default_redis', 'socket_connect_timeout') - }, - # 定时ping redis连接池，防止被服务端断开连接（s秒） - 'PING_INTERVAL': cfg.getint('default_redis', 'ping_interval') - } - }, -} - -settings.PASSWORD_HASHERS = [ - # 第一个元素为默认加密方式 - 'trest.utils.hasher.PBKDF2PasswordHasher', - 'trest.utils.hasher.PBKDF2SHA1PasswordHasher', -] - -#配置模版引擎 -#引入相应的TemplateLoader即可 -#若使用自带的请给予None -#支持mako和jinja2 -#mako设置为tornado.template.mako_loader.MakoTemplateLoader -#jinj2设置为tornado.template.jinja2_loader.Jinja2TemplateLoader -#初始化参数请参照jinja的Environment或mako的TemplateLookup,不再详细给出 -settings.TEMPLATE_CONFIG = storage({ - 'template_engine': None, - #模版路径由tornado.handler中commonhandler重写，无需指定，模版将存在于每个应用的根目录下 - 'filesystem_checks': True, #通用选项 - 'cache_directory': '../_tmpl_cache', #模版编译文件目录,通用选项 - 'collection_size': 50, #暂存入内存的模版项，可以提高性能，mako选项,详情见mako文档 - 'cache_size': 0, #类似于mako的collection_size，设定为-1为不清理缓存，0则每次都会重编译模板 - 'format_exceptions': False, #格式化异常输出，mako专用 - 'autoescape': False #默认转义设定，jinja2专用 -}) - -# tornado日志功能配置 -# Logging中有 -# NOTSET < DEBUG < INFO < WARNING < ERROR < CRITICAL 这几种级别， -# 日志会记录设置级别以上的日志 -# when 时间 按照哪种时间单位滚动（可选s-按秒，m-按分钟，h-按小时，d-按天，w0-w6-按指定的星期几，midnight-在午夜） -settings.LOGGING_DIR = os.path.join(ROOT_PATH, 'logs/') - -#其中name为getlogger指定的名字 -settings.standard_format = '[%(asctime)s][%(threadName)s:%(thread)d][task_id:%(name)s][%(filename)s:%(lineno)d]' \ - '[%(levelname)s][%(message)s]' -settings.LOGGING = ( - { - 'name': 'access_log', - 'log_to_stderr': True, - 'filename': 'access_log.log' - }, - { - 'name': 'tornado.debug.log', - 'level': 'DEBUG', - 'log_to_stderr': True, - 'when': 'w0', - 'interval': 1, - 'formatter': settings.standard_format, - 'filename': 'debug_log.log' - }, - { - 'name': 'tornado.info.log', - 'level': 'INFO', - 'log_to_stderr': True, - 'when': 'midnight', - 'interval': 1, - 'formatter': settings.standard_format, - 'filename': 'info_log.log' - }, - { - 'name': 'tornado.warning.log', - 'level': 'WARNING', - 'log_to_stderr': True, - 'when': 'midnight', - 'interval': 1, - 'formatter': settings.standard_format, - 'filename': 'warning_log.log' - }, - { - 'name': 'tornado.error.log', - 'level': 'ERROR', - 'log_to_stderr': True, - 'when': 'midnight', - 'interval': 1, - 'formatter': settings.standard_format, - 'filename': 'error_log.log' - }, - { - 'name': 'tornado.critical.log', - 'level': 'CRITICAL', - 'log_to_stderr': True, - 'when': 'midnight', - 'interval': 1, - 'formatter': settings.standard_format, - 'filename': 'critical_log.log' - }, -) diff --git a/trest/config/__init__.py b/trest/config/__init__.py new file mode 100644 index 0000000..de6efe2 --- /dev/null +++ b/trest/config/__init__.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import os +import sys +import yaml +from tornado.options import options + +from ..storage import dict_to_storage +from ..exception import ConfigError + +from .config import document + + +# 检查全局变量 ROOT_PATH 设置 +if hasattr(options, 'ROOT_PATH') and os.path.exists(options.ROOT_PATH): + ROOT_PATH = options.ROOT_PATH +else: + raise ConfigError('ROOT_PATH is not configured') + +with open(f'{ROOT_PATH}/.env', encoding='utf-8') as f: + cfg = yaml.safe_load(f) + +env = cfg.get('TREST_ENV','') +# 检查系统环境变量 TREST_ENV 设置 +if env not in ['local', 'dev', 'test', 'product']: + msg = f'The system variable TREST_ENV ({env}) is not one of the local, dev, test, or product' + raise ConfigError(msg) + +_yf = f'{ROOT_PATH}/configs/{env}.yaml' +if not(os.path.isfile(_yf) and os.access(_yf, os.R_OK)): + raise ConfigError(f'The ENV file({_yf}) does not exist or is unreadable') + +with open(_yf, encoding='utf-8') as f: + cfg = yaml.safe_load(f) + +_default_cfg = yaml.safe_load(document) +_default_cfg.update(cfg) +settings = dict_to_storage(_default_cfg) + +settings.ROOT_PATH = ROOT_PATH +settings.STATIC_PATH = os.path.join(ROOT_PATH, 'statics') +settings.ENV = env + +# 是否开启国际化 +settings.TRANSLATIONS_CONF = dict_to_storage({ + 'translations_dir': os.path.join(ROOT_PATH, 'datas/locales'), + 'locale_default': 'zh_CN', + 'use_accept_language': True +}) + +# tornado全局配置 +settings.TORNADO_CONF = { + 'xsrf_cookies': settings.xsrf_cookies, + 'login_url': '/admin/login', + 'cookie_secret': settings.cookie_secret, + # 'ui_modules': ui_modules, + 'template_path': os.path.join(ROOT_PATH, 'applications/admin/templates'), + 'static_path': settings.STATIC_PATH, + # 安全起见，可以定期生成新的cookie 秘钥，生成方法： + # base64.b64encode(uuid.uuid4().bytes + uuid.uuid4().bytes) +} + +# 系统角色，非超级管理员不允许编辑权限和删除 +settings.SYS_ROLE = [ + settings.SUPER_ROLE_ID, + settings.DEFAULT_ROLE_ID, +] + +log_dir = settings.log_cfg.log_dir +settings.LOGGING_DIR = log_dir if log_dir else os.path.join(ROOT_PATH, 'logs/') diff --git a/trest/config/config.py b/trest/config/config.py new file mode 100644 index 0000000..1d9c3b0 --- /dev/null +++ b/trest/config/config.py @@ -0,0 +1,226 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +document = """ +# f'{ROOT_PATH}/configs/{env}.yaml' +# 注意配置解析出来都是字符串，请不要带单引号或者双引号 +# 例如 '0.0.0.0' "0.0.0.0" 都会报错 + + +debug : true +xsrf_cookies : true +xheaders : true + +arbitrary_ip : 0.0.0.0 +port : 5080 +local_ip : 127.0.0.1 +translation : true +time_zone : Asia/Shanghai +language_code : zh-hans +cookie_secret : e921bfcd-ace4-4124-8657-c57a162365f6 +login_pwd_rsa_encrypt : True +default_aes_secret : 883d65f06fd447f3a1e69a36e73f58e0 +admin_session_key : de0b3fb0c2f44563944a8cccca7f225a +front_session_key : 171630947de24c969c28b2d178c4e0fe +valid_code_key : ab1195c6f0084b4f8b007d3aa7628a38 +token_key : f30a2331813f46d0adc2bcf26fcbbbf4 + + +INSTALLED_APPS : + - app1 + +rabbitmq_config : '' +sentry_url : +config_cache_prefix : 'conf:' +user_cache_prefix : 'user:' +admin_cache_prefix : 'admin:' + +# 超级管理员角色ID +SUPER_ROLE_ID : 1 +DEFAULT_ROLE_ID : 2 + +# Super Admin 必须是 int 型数据 +SUPER_ADMIN : + - 1 # admin uid + - 2 # + +# 是否开启国际化 +translation : true + +PASSWORD_HASHERS : + # 第一个元素为默认加密方式 + - 'trest.utils.hasher.PBKDF2PasswordHasher' + - 'trest.utils.hasher.PBKDF2SHA1PasswordHasher' + +# 中间件 # +# ########### +MIDDLEWARE_CLASSES : + - 'trest.middleware.dbalchemy.DBAlchemyMiddleware' + - 'trest.middleware.AccessLogMiddleware' + - 'trest.middleware.PushToMQMiddleware' + +session : + cache_alias : default_redis + name : nkzpg9NKBpKS2iaK + cookie_domain : + cookie_path : / + expires : 86400 + secret : fLjUfxqXtfNoIldA0A0J + version : v0.1.0 + +# sqlalchemy配置，列出部分，可自行根据sqlalchemy文档增加配置项 +# 该配置项对所有连接全局共享 +sqlalchemy : + # (s秒) + ping_db : 300 + # 每次取出ping多少个连接 + ping_conn_count : 5 + 'sqlalchemy.connect_args' : { + 'connect_timeout' : 3 + } + 'sqlalchemy.echo' : true + 'sqlalchemy.max_overflow' : 10 + 'sqlalchemy.echo_pool' : true + 'sqlalchemy.pool_timeout' : 5 + 'sqlalchemy.encoding' : utf8 + 'sqlalchemy.pool_size' : 5 + 'sqlalchemy.pool_recycle' : 3600 + # 手动指定连接池类 + 'sqlalchemy.poolclass' : QueuePool + +# 数据库连接字符串，元祖， +# 每组为n个数据库连接，有且只有一个master，可配与不配slave +DATABASE_CONNECTION : + default : + connections : + - + ROLE: 'master' + DRIVER : 'mysql+mysqldb' + UID : root + # 进过AES加密的密码，格式 aes::: + ciphertext + PASSWD : '123456' + HOST : '127.0.0.1' + PORT : 3306 + DATABASE : 'db_py_admin' + QUERY : {'charset' : 'utf8mb4'} + - + ROLE : 'slave' + DRIVER : 'mysql+mysqldb' + UID : root + # 进过AES加密的密码，格式 aes::: + ciphertext + PASSWD : '123456' + HOST : '127.0.0.1' + PORT : 3306 + DATABASE : 'db_py_admin' + QUERY : {'charset' : 'utf8mb4'} + +########### +# 缓存配置 # +########### +CACHES : + 'default': + 'BACKEND': 'trest.cache.backends.localcache.LocMemCache' + 'LOCATION': 'process_cache' + 'OPTIONS': + 'MAX_ENTRIES': 10000 + 'CULL_FREQUENCY': 3 + + 'default_redis': + 'BACKEND': 'trest.cache.backends.rediscache.RedisCache' + 'LOCATION': '127.0.0.1:6379' + 'OPTIONS': + 'DB': 0 + 'PASSWORD': '' + 'PARSER_CLASS': 'redis.connection.DefaultParser' + # 定时ping redis连接池，防止被服务端断开连接（s秒） + 'PING_INTERVAL': 120 + 'POOL_KWARGS': + 'socket_timeout': 2 + 'socket_connect_timeout': 2 + +# 配置模版引擎 +# 引入相应的TemplateLoader即可 +# 若使用自带的请留空 +# 支持mako和jinja2 +# mako设置为 tornado.template.mako_loader.MakoTemplateLoader +# jinj2设置为 tornado.template.jinja2_loader.Jinja2TemplateLoader +# 初始化参数请参照jinja的Environment或mako的TemplateLookup,不再详细给出 +TEMPLATE_CFG : + 'template_engine': '' + # 模版路径由tornado.handler中commonhandler重写， + # 无需指定，模版将存在于每个应用的根目录下 + #通用选项 + 'filesystem_checks': True + 'cache_directory': '../_tmpl_cache' # 模版编译文件目录,通用选项 + # 暂存入内存的模版项，可以提高性能，mako选项,详情见mako文档 + 'collection_size': 50 + # 类似于mako的collection_size，设定为-1为不清理缓存，0则每次都会重编译模板 + 'cache_size': 0 + # 格式化异常输出，mako专用 + 'format_exceptions': False + # 默认转义设定，jinja2专用 + 'autoescape': False + +# tornado日志功能配置 +# Logging中有 +# NOTSET < DEBUG < INFO < WARNING < ERROR < CRITICAL 这几种级别， +# 日志会记录设置级别以上的日志 +# when 时间 按照哪种时间单位滚动（可选s-按秒，m-按分钟，h-按小时，d-按天，w0-w6-按指定的星期几，midnight-在午夜） +log_cfg: + log_dir: '' + standard_format : &standard_format '[%(asctime)s][%(threadName)s:%(thread)d][task_id:%(name)s][%(filename)s:%(lineno)d] [%(levelname)s][%(message)s]' + + logging : + - + 'name': 'access_log' + 'log_to_stderr': True + 'filename': 'access_log.log' + - + 'name': 'tornado.debug.log' + 'level': 'DEBUG' + 'log_to_stderr': True + 'when': 'w0' + 'interval': 1 + 'formatter': *standard_format + 'filename': 'debug_log.log' + - + 'name': 'tornado.info.log' + 'level': 'INFO' + 'log_to_stderr': True + 'when': 'midnight' + 'interval': 1 + 'formatter': *standard_format + 'filename': 'info_log.log' + - + 'name': 'tornado.warning.log' + 'level': 'WARNING' + 'log_to_stderr': True + 'when': 'midnight' + 'interval': 1 + 'formatter': *standard_format + 'filename': 'warning_log.log' + - + 'name': 'tornado.error.log' + 'level': 'ERROR' + 'log_to_stderr': True + 'when': 'midnight' + 'interval': 1 + 'formatter': *standard_format + 'filename': 'error_log.log' + - + 'name': 'tornado.critical.log' + 'level': 'CRITICAL' + 'log_to_stderr': True + 'when': 'midnight' + 'interval': 1 + 'formatter': *standard_format + 'filename': 'critical_log.log' + +email : + from_name : + from_addr : + smtp_sever : + smtp_port : + auth_code : + +""" diff --git a/trest/db/dbalchemy.py b/trest/db/dbalchemy.py index af31eb0..d42c649 100644 --- a/trest/db/dbalchemy.py +++ b/trest/db/dbalchemy.py @@ -18,7 +18,7 @@ from trest.config import settings from ..logger import SysLogger from ..exception import ConfigError -from ..storage import storage +from ..storage import Storage from ..utils.encrypter import aes_decrypt @@ -38,7 +38,9 @@ ) _BASE_SQLALCHEMY_CONFIGURATION = { - 'sqlalchemy.connect_args': {}, + 'sqlalchemy.connect_args': { + 'connect_timeout': 3, + }, 'sqlalchemy.echo': False, 'sqlalchemy.max_overflow': 10, 'sqlalchemy.echo_pool': False, @@ -124,16 +126,16 @@ def parser_engines(cls): @classmethod def parser_sqlalchemy_conf(cls): try: - config = settings.SQLALCHEMY_CONFIGURATION - except ConfigError as ex: - SysLogger.warning("SQLALCHEMY_CONFIGURATION not found,using default sqlalchemy configuration") + config = settings.sqlalchemy + except Exception as ex: + SysLogger.warning("sqlalchemy not found,using default sqlalchemy configuration") config = _BASE_SQLALCHEMY_CONFIGURATION - poolclass_conf = _SQLALCHEMY_PREFIX + 'poolclass' - poolclass = config[poolclass_conf] if poolclass_conf in config else None - + poolclass_cfg = _SQLALCHEMY_PREFIX + 'poolclass' + poolclass = config[poolclass_cfg] if poolclass_cfg in config else None + # print("poolclass_cfg ", poolclass_cfg) if poolclass: - config[poolclass_conf] = import_object('sqlalchemy.pool.' + poolclass) + config[poolclass_cfg] = import_object('sqlalchemy.pool.' + poolclass) return config @@ -155,7 +157,7 @@ def conn_pool(self): else: # 创建连接 with _Connector._conn_lock: - connection_pool = storage() + connection_pool = Storage() engines = DBConfigParser.parser_engines() config = DBConfigParser.parser_sqlalchemy_conf() try: diff --git a/trest/middleware/dbalchemy.py b/trest/middleware/dbalchemy.py index 664f2a5..bf2994e 100644 --- a/trest/middleware/dbalchemy.py +++ b/trest/middleware/dbalchemy.py @@ -33,8 +33,7 @@ def ping_connection(dbapi_connection, connection_record, connection_proxy): def ping_db(conn_, ping_inteval): @coroutine def ping_func(): - ping_conn_count = settings.sqlalchemy.get('ping_conn_count', 5) - ping_conn_count = int(ping_conn_count) + ping_conn_count = settings.sqlalchemy.ping_conn_count yield [conn_.ping_db() for _ in range(ping_conn_count)] PeriodicCallback(ping_func, ping_inteval * 1000).start() @@ -42,20 +41,12 @@ def ping_func(): class DBAlchemyMiddleware(object): connection = {} def process_init(self, application): - configed_db = False - try: - configed_db = settings.DATABASE_CONNECTION['default']['connections'][0]['HOST'] - configed_db = True if configed_db else False - except Exception as e: - pass - if settings.sqlalchemy.get('ping_db') and configed_db: - self.connection = Connector.conn_pool - connection_event() - # 定时ping数据库，防止mysql go away，定时检测防丢 - interval = int(settings.sqlalchemy.ping_db) - if interval > 0: - for k, conn in self.connection.items(): - ping_db(conn, interval) + self.connection = Connector.conn_pool + connection_event() + # 定时ping数据库，防止mysql go away，定时检测防丢 + if settings.sqlalchemy.ping_db > 0: + for k, conn in self.connection.items(): + ping_db(conn, settings.sqlalchemy.ping_db) def process_response(self, handler, clear, chunk): """ diff --git a/trest/middleware/response.py b/trest/middleware/response.py index f226089..0c6ff9d 100644 --- a/trest/middleware/response.py +++ b/trest/middleware/response.py @@ -21,7 +21,7 @@ def process_response(self, handler, clear, chunk): 响应内容，chunk为携带响内容的list，你不可以直接对chunk赋值， 可以通过chunk[index]来改写响应内容，或再次执行handler.write() """ - if settings.sys.get('rabbitmq_config', None) and handler.response_to_mq: + if settings.get('rabbitmq_config', None) and handler.response_to_mq: data = handler.request.arguments # print('chunk', type(chunk), chunk) try: diff --git a/trest/storage.py b/trest/storage.py index c06da94..2e3e82f 100644 --- a/trest/storage.py +++ b/trest/storage.py @@ -29,9 +29,6 @@ def __delattr__(self, key): def __repr__(self): return '' - -storage = Storage - class SortedDict(dict): """ A dictionary that keeps its keys in the order in which they're inserted. @@ -162,6 +159,26 @@ def clear(self): super(SortedDict, self).clear() self.keyOrder = [] +def _dict_to_storage_check_list(from_list): + for i,item in enumerate(from_list): + if type(item) == dict: + from_list[i] = dict_to_storage(item) + elif type(item) == list: + from_list[i] = _dict_to_storage_check_list(item) + return from_list -sorteddict = SortedDict +def dict_to_storage(from_dict): + """ + >>> b = {'a': [ [ [{'a3':3}]]]} + >>> a = {'a': [ [ [{'a3':3, 'b': b}]]]} + dict_to_storage(a).a[0][0][0].b.a[0][0][0].a3 == 3 + Returns: + [type] -- [description] + """ + for k in from_dict.keys(): + if type(from_dict[k]) == dict: + from_dict[k] = dict_to_storage(from_dict[k]) + elif type(from_dict[k]) == list: + from_dict[k] = _dict_to_storage_check_list(from_dict[k]) + return Storage(from_dict) diff --git a/trest/utils/encrypter.py b/trest/utils/encrypter.py index d8ba563..7b0f100 100644 --- a/trest/utils/encrypter.py +++ b/trest/utils/encrypter.py @@ -181,7 +181,7 @@ def decrypt(self, text): def aes_decrypt(ciphertext, secret=None, prefix='aes:::'): - secret = secret if secret else settings.sys.default_aes_secret + secret = secret if secret else settings.default_aes_secret cipher = AESEncrypter(secret) prefix_len = len(prefix) if ciphertext[0:prefix_len]==prefix: @@ -190,7 +190,7 @@ def aes_decrypt(ciphertext, secret=None, prefix='aes:::'): return ciphertext def aes_encrypt(plaintext, secret=None, prefix='aes:::'): - secret = secret if secret else settings.sys.default_aes_secret + secret = secret if secret else settings.default_aes_secret cipher = AESEncrypter(secret) encrypted = cipher.encrypt(plaintext) return '%s%s' % (prefix, encrypted) diff --git a/trest/utils/utime.py b/trest/utils/utime.py index 2de6976..f1537fb 100644 --- a/trest/utils/utime.py +++ b/trest/utils/utime.py @@ -28,7 +28,7 @@ def dt_now(to_tz='UTC'): Returns: datetime.datetime -- 例如 datetime.datetime(2018, 5, 8, 3, 50, 50, 356945, tzinfo=) """ - to_tz = settings.sys.time_zone if to_tz is None else to_tz + to_tz = settings.time_zone if to_tz is None else to_tz tz = pytz.timezone(to_tz) return datetime.datetime.now(tz) @@ -45,7 +45,7 @@ def dt_to_timezone(dt, to_tz='UTC'): """ if not isinstance(dt, datetime.datetime): return dt - to_tz = settings.sys.time_zone if to_tz is None else to_tz + to_tz = settings.time_zone if to_tz is None else to_tz tz = pytz.timezone(to_tz) return dt.astimezone(tz) @@ -53,7 +53,7 @@ def ts_to_datetime(ts, to_tz='UTC'): """ ts默认为Unix时间戳，而不是本地时间戳 """ - to_tz = settings.sys.time_zone if to_tz is None else to_tz + to_tz = settings.time_zone if to_tz is None else to_tz dt = datetime.datetime.fromtimestamp(ts) return dt_to_timezone(dt, to_tz) @@ -63,7 +63,7 @@ def ts_to_str(ts, fmt=None, to_tz='UTC'): fmt='%Y-%m-%d %H-%M-%S %Z' '2019-04-16 07-06-24 UTC' fmt=None '2019-04-16 07:06:24+00:00 """ - to_tz = settings.sys.time_zone if to_tz is None else to_tz + to_tz = settings.time_zone if to_tz is None else to_tz dt = ts_to_datetime(ts, to_tz) return str(dt) if fmt is None else dt.strftime(fmt) @@ -71,7 +71,7 @@ def datetime_to_ts(dt, to_tz='UTC'): """ datetime转换为时间戳，默认转换为Unix时间戳 """ - to_tz = settings.sys.time_zone if to_tz is None else to_tz + to_tz = settings.time_zone if to_tz is None else to_tz dt2 = dt_to_timezone(dt, to_tz) return dt2.timestamp() @@ -92,7 +92,7 @@ def str_to_datetime(dt_str, to_tz='UTC'): Returns: [datetime.datetime] -- [description] """ - to_tz = settings.sys.time_zone if to_tz is None else to_tz + to_tz = settings.time_zone if to_tz is None else to_tz tz = pytz.timezone(to_tz) dt = dateutil.parser.parse(dt_str) dt = dt.astimezone(tz) diff --git a/trest/webserver.py b/trest/webserver.py index 334cadd..f508eb0 100644 --- a/trest/webserver.py +++ b/trest/webserver.py @@ -73,7 +73,7 @@ def _install_application(self, handlers): middlewares=settings.MIDDLEWARE_CLASSES, **tornado_conf) app_obj.sentry_client = AsyncSentryClient( - settings.sys.get('sentry_url', '') + settings.get('sentry_url', '') ) return app_obj @@ -130,11 +130,11 @@ def start(self): def _print_settings_info(self): if settings.debug: print('tornado version: %s' % tornado.version) - print('locale support: %s' % settings.sys.get('translation', False)) + print('locale support: %s' % settings.get('translation', False)) print('load apps:') for app in settings.INSTALLED_APPS: print(' - %s' % str(app)) - print('template engine: %s' % (settings.TEMPLATE_CONFIG.template_engine or 'default')) + print('template engine: %s' % (settings.TEMPLATE_CFG.template_engine or 'default')) print('server started. development server at http://%s:%s/' % (options.address, options.port)) def _parse_command(self, args=None, final=False): @@ -157,7 +157,7 @@ def _parse_logger_callback(self): tornado_logger = logging.getLogger('tornado') enable_pretty_logging(logger=tornado_logger) logdir = options.logging_dir or settings.LOGGING_DIR - for log in settings.LOGGING: + for log in settings.log_cfg.logging: opt = OptionParser() define_logging_options(opt) self._define(opt) @@ -200,8 +200,8 @@ def _define(self, options=options): except: pass - address = settings.sys.arbitrary_ip - options.define("port", default=settings.sys.port, help="run server on it", type=int) + address = settings.arbitrary_ip + options.define("port", default=settings.port, help="run server on it", type=int) options.define("settings", default='', help="setting module name", type=str) options.define("address", default=address, help=f'listen host,default:{address}', type=str) options.define("log_patch", default=True,