diff --git a/Dockerfile b/Dockerfile index 9081d18..b488085 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,34 +1,12 @@ -FROM centos:7 -# 设置编码 -ENV LANG en_US.UTF-8 -# 同步时间 -ENV TZ=Asia/Shanghai -RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone +FROM registry.cn-hangzhou.aliyuncs.com/sourcegarden/python:centos7-3.6 -# 1. 安装基本依赖 -RUN yum update -y && yum install epel-release -y && yum update -y && yum install wget unzip epel-release nginx xz gcc automake zlib-devel openssl-devel supervisor groupinstall development libxslt-devel libxml2-devel libcurl-devel git -y -#WORKDIR /var/www/ - -# 2. 准备python -RUN wget https://www.python.org/ftp/python/3.6.6/Python-3.6.6.tar.xz -RUN xz -d Python-3.6.6.tar.xz && tar xvf Python-3.6.6.tar && cd Python-3.6.6 && ./configure && make && make install - -# 3. 安装websdk -RUN pip3 install --upgrade pip -RUN pip3 install -U git+https://github.com/ss1917/ops_sdk.git -# 4. 复制代码 -RUN mkdir -p /var/www/ ADD . /var/www/codo-cmdb/ -# 5. 安装pip依赖 -RUN pip3 install -r /var/www/codo-cmdb/doc/requirements.txt - -# 6. 日志 -VOLUME /var/log/ +RUN pip3 install -i https://pypi.tuna.tsinghua.edu.cn/simple -r /var/www/codo-cmdb/requirements.txt -# 7. 准备文件 -COPY doc/nginx_ops.conf /etc/nginx/conf.d/default.conf -COPY doc/supervisor_ops.conf /etc/supervisord.conf +COPY docker/nginx_default.conf /etc/nginx/nginx.conf +COPY docker/nginx_ops.conf /etc/nginx/conf.d/default.conf +COPY docker/supervisor_ops.conf /etc/supervisord.conf EXPOSE 80 CMD ["/usr/bin/supervisord"] \ No newline at end of file diff --git a/README.md b/README.md index 487cf40..cc1ac13 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,18 @@ > 基于Python Tornado实现的一套CMDB资产管理系统，前端使用Vue+Iview-admin +### 更新 + +- beta0.3.1 支持了华为云资产获取 + +``` +前端更换成最新的前端、更改表结构即可 +ALTER TABLE `asset_configs` ADD `project_id` VARCHAR(120) NOT NULL ; +ALTER TABLE `asset_configs` ADD `huawei_cloud` VARCHAR(120) NOT NULL ; +ALTER TABLE `asset_configs` ADD `huawei_instance_id` VARCHAR(120) NOT NULL ; + +#若是不想更改表结构、或者是第一次部署的同学，默认docker exec -ti cmdb_codo_cmdb_1 /usr/local/bin/python3 /var/www/codo-cmdb/db_sync.py 初始化表结构即可 +``` **目前功能** diff --git a/biz/crontab_app.py b/biz/crontab_app.py index 8605375..0c9fd76 100644 --- a/biz/crontab_app.py +++ b/biz/crontab_app.py @@ -1,23 +1,23 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# @Time : 2019/4/26 14:30 -# @Author : Fred Yangxiaofei -# @File : auto_update_app.py -# @Role : 一些定时执行的程序 - - -import tornado -from websdk.application import Application as myApplication -from biz.timed_program import tail_data as my_timed_program - - -class Application(myApplication): - def __init__(self, **settings): - urls = [] - my_program_callback = tornado.ioloop.PeriodicCallback(my_timed_program, 3600000) #1小时 - my_program_callback.start() - super(Application, self).__init__(urls, **settings) - - -if __name__ == '__main__': - pass +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Time : 2019/4/26 14:30 +# @Author : Fred Yangxiaofei +# @File : auto_update_app.py +# @Role : 一些定时执行的程序 + + +import tornado +from websdk.application import Application as myApplication +from biz.timed_program import tail_data as my_timed_program + + +class Application(myApplication): + def __init__(self, **settings): + urls = [] + my_program_callback = tornado.ioloop.PeriodicCallback(my_timed_program, 3600000) #1小时 + my_program_callback.start() + super(Application, self).__init__(urls, **settings) + + +if __name__ == '__main__': + pass diff --git a/biz/handlers/admin_user_handler.py b/biz/handlers/admin_user_handler.py index 8830688..9766348 100644 --- a/biz/handlers/admin_user_handler.py +++ b/biz/handlers/admin_user_handler.py @@ -1,102 +1,102 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# @Time : 2019/4/18 10:12 -# @Author : Fred Yangxiaofei -# @File : admin_user_handler.py -# @Role : 管理用户 - - -import json -from libs.base_handler import BaseHandler -from models.server import AdminUser, model_to_dict -from websdk.db_context import DBContext - - -class AdminUserHandler(BaseHandler): - def get(self, *args, **kwargs): - key = self.get_argument('key', default=None, strip=True) - value = self.get_argument('value', default=None, strip=True) - page_size = self.get_argument('page', default=1, strip=True) - limit = self.get_argument('limit', default=15, strip=True) - limit_start = (int(page_size) - 1) * int(limit) - admin_user_list = [] - with DBContext('w') as session: - if key and value: - count = session.query(AdminUser).filter_by(**{key: value}).count() - admin_user_data = session.query(AdminUser).filter_by(**{key: value}).order_by( - AdminUser.id).offset(limit_start).limit(int(limit)) - else: - count = session.query(AdminUser).count() - admin_user_data = session.query(AdminUser).order_by(AdminUser.id).offset( - limit_start).limit(int(limit)) - - for data in admin_user_data: - data_dict = model_to_dict(data) - data_dict['update_time'] = str(data_dict['update_time']) - admin_user_list.append(data_dict) - return self.write(dict(code=0, msg='获取成功', count=count, data=admin_user_list)) - - def post(self, *args, **kwargs): - data = json.loads(self.request.body.decode("utf-8")) - admin_user = data.get('admin_user', None) - system_user = data.get('system_user', None) - # password = data.get('password', None) - user_key = data.get('user_key', None) - remarks = data.get('remarks', None) - - if not admin_user or not system_user or not user_key: - return self.write(dict(code=-2, msg='关键参数不能为空')) - - with DBContext('r') as session: - exist_id = session.query(AdminUser.id).filter(AdminUser.admin_user == admin_user).first() - count = session.query(AdminUser).count() - print(count) - if exist_id: - return self.write(dict(code=-2, msg='不要重复记录')) - - if count > 15: - return self.write(dict(code=-2, msg='管理用户最大只允许15个')) - - with DBContext('w', None, True) as session: - new_admin_user = AdminUser(admin_user=admin_user, system_user=system_user, user_key=user_key, remarks=remarks) - session.add(new_admin_user) - return self.write(dict(code=0, msg='添加成功')) - - def put(self, *args, **kwargs): - data = json.loads(self.request.body.decode("utf-8")) - admin_user_id = data.get('id') - admin_user = data.get('admin_user', None) - system_user = data.get('system_user', None) - # password = data.get('password', None) - user_key = data.get('user_key', None) - remarks = data.get('remarks', None) - - if not admin_user or not system_user or not user_key: - return self.write(dict(code=-2, msg='关键参数不能为空')) - - update_info = { - "admin_user": admin_user, - "system_user": system_user, - "user_key": user_key, - "remarks": remarks, - } - - with DBContext('w', None, True) as session: - session.query(AdminUser).filter(AdminUser.id == admin_user_id).update(update_info) - self.write(dict(code=0, msg='更新成功')) - - def delete(self, *args, **kwargs): - data = json.loads(self.request.body.decode("utf-8")) - id = data.get('id') - if not id: - return self.write(dict(code=-2, msg='关键参数不能为空')) - - with DBContext('w', None, True) as session: - session.query(AdminUser).filter(AdminUser.id == id).delete(synchronize_session=False) - - self.write(dict(code=0, msg='删除成功')) - - -admin_user_urls = [ - (r"/v1/cmdb/admin_user/", AdminUserHandler) -] +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Time : 2019/4/18 10:12 +# @Author : Fred Yangxiaofei +# @File : admin_user_handler.py +# @Role : 管理用户 + + +import json +from libs.base_handler import BaseHandler +from models.server import AdminUser, model_to_dict +from websdk.db_context import DBContext + + +class AdminUserHandler(BaseHandler): + def get(self, *args, **kwargs): + key = self.get_argument('key', default=None, strip=True) + value = self.get_argument('value', default=None, strip=True) + page_size = self.get_argument('page', default=1, strip=True) + limit = self.get_argument('limit', default=15, strip=True) + limit_start = (int(page_size) - 1) * int(limit) + admin_user_list = [] + with DBContext('w') as session: + if key and value: + count = session.query(AdminUser).filter_by(**{key: value}).count() + admin_user_data = session.query(AdminUser).filter_by(**{key: value}).order_by( + AdminUser.id).offset(limit_start).limit(int(limit)) + else: + count = session.query(AdminUser).count() + admin_user_data = session.query(AdminUser).order_by(AdminUser.id).offset( + limit_start).limit(int(limit)) + + for data in admin_user_data: + data_dict = model_to_dict(data) + data_dict['update_time'] = str(data_dict['update_time']) + admin_user_list.append(data_dict) + return self.write(dict(code=0, msg='获取成功', count=count, data=admin_user_list)) + + def post(self, *args, **kwargs): + data = json.loads(self.request.body.decode("utf-8")) + admin_user = data.get('admin_user', None) + system_user = data.get('system_user', None) + # password = data.get('password', None) + user_key = data.get('user_key', None) + remarks = data.get('remarks', None) + + if not admin_user or not system_user or not user_key: + return self.write(dict(code=-2, msg='关键参数不能为空')) + + with DBContext('r') as session: + exist_id = session.query(AdminUser.id).filter(AdminUser.admin_user == admin_user).first() + count = session.query(AdminUser).count() + print(count) + if exist_id: + return self.write(dict(code=-2, msg='不要重复记录')) + + if count > 15: + return self.write(dict(code=-2, msg='管理用户最大只允许15个')) + + with DBContext('w', None, True) as session: + new_admin_user = AdminUser(admin_user=admin_user, system_user=system_user, user_key=user_key, remarks=remarks) + session.add(new_admin_user) + return self.write(dict(code=0, msg='添加成功')) + + def put(self, *args, **kwargs): + data = json.loads(self.request.body.decode("utf-8")) + admin_user_id = data.get('id') + admin_user = data.get('admin_user', None) + system_user = data.get('system_user', None) + # password = data.get('password', None) + user_key = data.get('user_key', None) + remarks = data.get('remarks', None) + + if not admin_user or not system_user or not user_key: + return self.write(dict(code=-2, msg='关键参数不能为空')) + + update_info = { + "admin_user": admin_user, + "system_user": system_user, + "user_key": user_key, + "remarks": remarks, + } + + with DBContext('w', None, True) as session: + session.query(AdminUser).filter(AdminUser.id == admin_user_id).update(update_info) + self.write(dict(code=0, msg='更新成功')) + + def delete(self, *args, **kwargs): + data = json.loads(self.request.body.decode("utf-8")) + id = data.get('id') + if not id: + return self.write(dict(code=-2, msg='关键参数不能为空')) + + with DBContext('w', None, True) as session: + session.query(AdminUser).filter(AdminUser.id == id).delete(synchronize_session=False) + + self.write(dict(code=0, msg='删除成功')) + + +admin_user_urls = [ + (r"/v1/cmdb/admin_user/", AdminUserHandler) +] diff --git a/biz/handlers/asset_configs_handler.py b/biz/handlers/asset_configs_handler.py index 60792c9..18662b9 100644 --- a/biz/handlers/asset_configs_handler.py +++ b/biz/handlers/asset_configs_handler.py @@ -1,448 +1,448 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# @Time : 2019/5/14 10:30 -# @Author : Fred Yangxiaofei -# @File : asset_configs_handler.py -# @Role : 资产配置，主要用来实现AWS/aliyun/qcloud 多账户云机器自动录入CMDB - -import json -import time -import openstack -import keystoneauth1 -import tornado.web -from tornado import gen -from concurrent.futures import ThreadPoolExecutor -from tornado.concurrent import run_on_executor - -from models.server import AssetConfigs, model_to_dict -from websdk.db_context import DBContext -from websdk.web_logs import ins_log -from opssdk.operate import MyCryptV2 -from libs.base_handler import BaseHandler -from libs.aws.ec2 import Ec2Api -from libs.aws.rds import RDSApi as AwsRdsApi -from libs.aws.ec2 import main as aws_ec2_update -from libs.aws.rds import main as aws_rds_update -from libs.aws.elasticache import main as aws_cache_update -from libs.aws.elasticache import CacheApi as AwsCacheApi -from libs.aliyun.ecs import EcsAPi -from libs.aliyun.rds import RdsApi as AliyunRdsApi -from libs.aliyun.redis import RedisApi as AliyunRedisApi -from libs.aliyun.ecs import main as aliyun_ecs_update -from libs.aliyun.rds import main as aliyun_rds_update -from libs.aliyun.redis import main as aliyun_redis_update -from libs.qcloud.cvm import CVMApi -from libs.qcloud.cdb import CDBApi as QcloudCdbApi -from libs.qcloud.redis import RedisApi -from libs.qcloud.cvm import main as qcloud_cvm_update -from libs.qcloud.cdb import main as qcloud_cdb_update -from libs.qcloud.redis import main as qcloud_redis_update -from libs.huaweiyun.huawei_ecs import HuaweiEcsApi -from libs.huaweiyun.huawei_ecs import main as huawei_ecs_update -from ucloud.core import exc -from libs.ucloud.uhost import UHostAPI -from libs.ucloud.uhost import main as ucloud_uhost_update -from libs.ucloud.udb import UdbAPI -from libs.ucloud.udb import main as ucloud_udb_update -from libs.ucloud.uredis import UredisAPI -from libs.ucloud.uredis import main as ucloud_uredis_update - -mc = MyCryptV2() # 实例化 - - -class AssetConfigsHandler(BaseHandler): - def get(self, *args, **kwargs): - key = self.get_argument('key', default=None, strip=True) - value = self.get_argument('value', default=None, strip=True) - asset_configs_list = [] - with DBContext('r') as session: - if key and value: - asset_configs_data = session.query(AssetConfigs).filter_by(**{key: value}).all() - else: - asset_configs_data = session.query(AssetConfigs).all() - - for data in asset_configs_data: - data_dict = model_to_dict(data) - data_dict['create_time'] = str(data_dict['create_time']) - data_dict['update_time'] = str(data_dict['update_time']) - # 解密AccessKey - if data_dict.get('access_key'): - data_dict['access_key'] = mc.my_decrypt(data_dict.get('access_key')) - asset_configs_list.append(data_dict) - - return self.write(dict(code=0, msg='获取成功', data=asset_configs_list)) - - def post(self, *args, **kwargs): - data = json.loads(self.request.body.decode("utf-8")) - name = data.get('name', None) - account = data.get('account', None) - region = data.get('region', None).strip() - access_id = data.get('access_id', None).strip() - access_key = data.get('access_key', None).strip() - default_admin_user = data.get('default_admin_user', None) - state = data.get('state', None) - remarks = data.get('remarks', None) - # 华为云需要额外三个数据 - project_id = data.get('project_id', 'Null') - huawei_cloud = data.get('huawei_cloud', 'Null') - huawei_instance_id = data.get('huawei_instance_id', 'Null') - - if not name or not account or not region or not access_id or not access_key or not state: - return self.write(dict(code=-2, msg='关键参数不能为空')) - - with DBContext('r', None, True) as session: - exist_id = session.query(AssetConfigs.id).filter(AssetConfigs.name == name).first() - - if exist_id: - return self.write(dict(code=-2, msg='不要重复记录')) - - # 对密钥进行加密再写数据库 - _access_key = mc.my_encrypt(access_key) - - with DBContext('w', None, True) as session: - new_asset_config = AssetConfigs(name=name, account=account, region=region, access_id=access_id, - access_key=_access_key, default_admin_user=default_admin_user, state=state, - remarks=remarks, project_id=project_id, huawei_cloud=huawei_cloud, - huawei_instance_id=huawei_instance_id) - session.add(new_asset_config) - - return self.write(dict(code=0, msg='添加成功')) - - def put(self, *args, **kwargs): - data = json.loads(self.request.body.decode("utf-8")) - id = data.get('id', None) - name = data.get('name', None) - account = data.get('account', None) - region = data.get('region', None) - access_id = data.get('access_id', None) - access_key = data.get('access_key', None) - default_admin_user = data.get('default_admin_user', None) - state = data.get('state', None) - remarks = data.get('remarks', None) - # 华为云需要额外三个数据 - project_id = data.get('project_id', None) - huawei_cloud = data.get('huawei_cloud', None) - huawei_instance_id = data.get('huawei_instance_id', None) - - if not name or not account or not region or not access_id or not access_key or not state: - return self.write(dict(code=-2, msg='关键参数不能为空')) - - # 对密钥进行加密再写数据库 - _access_key = mc.my_encrypt(access_key) - with DBContext('w', None, True) as session: - session.query(AssetConfigs).filter(AssetConfigs.id == id).update( - {AssetConfigs.account: account, AssetConfigs.access_id: access_id, AssetConfigs.access_key: _access_key, - AssetConfigs.default_admin_user: default_admin_user, AssetConfigs.state: state, - AssetConfigs.remarks: remarks, AssetConfigs.project_id: project_id, - AssetConfigs.huawei_cloud: huawei_cloud, AssetConfigs.huawei_instance_id: huawei_instance_id}) - return self.write(dict(code=0, msg='编辑成功')) - - def patch(self, *args, **kwargs): - '''开关控制，开启/禁用''' - data = json.loads(self.request.body.decode("utf-8")) - id = data.get('id') - - msg = 'Not Fount!' - - if not id: - return self.write(dict(code=-2, msg='关键参数不能为空')) - - with DBContext('r') as session: - config_state = session.query(AssetConfigs.state).filter(AssetConfigs.id == id).first() - - if not config_state: - return self.write(dict(code=0, msg=msg)) - - if config_state[0] == 'false': - msg = '启用成功' - new_state = 'true' - - elif config_state[0] == 'true': - msg = '禁用成功' - new_state = 'false' - - with DBContext('w', None, True) as session: - # print(new_state) - session.query(AssetConfigs).filter(AssetConfigs.id == id).update({AssetConfigs.state: new_state}) - - return self.write(dict(code=0, msg=msg)) - - def delete(self, *args, **kwargs): - data = json.loads(self.request.body.decode("utf-8")) - id = data.get('id') - if not id: - return self.write(dict(code=-2, msg='关键参数不能为空')) - - with DBContext('w', None, True) as session: - session.query(AssetConfigs).filter(AssetConfigs.id == id).delete(synchronize_session=False) - - self.write(dict(code=0, msg='删除成功')) - - -class ApiPermissionHandler(BaseHandler): - '''测试用户填写的信息及认证是否正确,防止主进程卡死，使用异步方法测试''' - _thread_pool = ThreadPoolExecutor(3) - - @run_on_executor(executor='_thread_pool') - def server_api(self, account, region, access_id, access_key, project_id, huawei_cloud, huawei_instance_id): - """ - 测试各厂商server API的权限 - :return: - """ - # 收集测试错误日志 - err_msg = '' - # 测试接口的时候默认管理用户不需要 - default_admin_user = '' - if account == 'AWS': - ins_log.read_log('info', 'AWS EC2 API TEST') - obj = Ec2Api(access_id, access_key, region, default_admin_user) - try: - obj.test_auth() - except Exception as e: - err_msg = 'Error：{}'.format(e) - elif account == '阿里云': - ins_log.read_log('info', 'Aliyun ECS API TEST') - obj = EcsAPi(access_id, access_key, region, default_admin_user) - try: - obj.test_auth() - except Exception as e: - err_msg = 'Error：{}'.format(e) - - elif account == '腾讯云': - ins_log.read_log('info', 'Qcloud CVM API TEST') - obj = CVMApi(access_id, access_key, region, default_admin_user) - try: - result_data = obj.test_auth() - if result_data['Response'].get('Error'): - err_msg = 'Error：{}'.format(result_data['Response']) - except Exception as e: - ins_log.read_log('error', e) - - elif account == '华为云': - ins_log.read_log('info', 'HuaweiCloud ECS API TEST') - obj = HuaweiEcsApi(access_id=access_id, access_key=access_key, region=region, cloud=huawei_cloud, - project_id=project_id, - default_admin_user=default_admin_user) - try: - obj.test_auth(huawei_instance_id) - except keystoneauth1.exceptions.http.Unauthorized: - err_msg = '请检查AccessID和AccessKey权限是否正确' - except openstack.exceptions.HttpException as err: - err_msg = 'openstack error {}'.format(err) - except Exception as e: - err_msg = 'error: {}'.format(e) - elif account == 'UCloud': - ins_log.read_log('info', 'UCloud Uhost API TEST') - try: - UHostAPI.permission_auth(access_id=access_id, access_key=access_key, region=region, - project_id=project_id) - except exc.UCloudException as uerr: - err_msg = 'ucloud error: {ucloud_error}'.format(ucloud_error=uerr) - except Exception as e: - err_msg = 'error: {}'.format(e) - else: - err_msg = 'In Development.' - return err_msg - - @run_on_executor(executor='_thread_pool') - def rds_api(self, account, region, access_id, access_key, project_id): - """ - 测试DB API权限 - :return: - """ - # 错误收集 - err_msg = '' - if account == 'AWS': - ins_log.read_log('info', 'AWS RDS API TEST') - obj = AwsRdsApi(access_id, access_key, region, ) - try: - obj.test_auth() - except Exception as e: - err_msg = 'Error：{}'.format(e) - elif account == '阿里云': - ins_log.read_log('info', 'Aliyun RDS API TEST') - obj = AliyunRdsApi(access_id, access_key, region) - try: - obj.test_auth() - except Exception as e: - err_msg = 'Error：{}'.format(e) - - elif account == '腾讯云': - ins_log.read_log('info', 'Qcloud CDB API TEST') - obj = QcloudCdbApi(access_id, access_key, region) - try: - result_data = obj.test_auth() - if result_data['Response'].get('Error'): - err_msg = 'Error：{}'.format(result_data['Response']) - except Exception as e: - ins_log.read_log('error', e) - elif account == 'UCloud': - ins_log.read_log('info', 'UCloud UDB API TEST') - try: - UdbAPI.permission_auth(access_id=access_id, access_key=access_key, region=region, - project_id=project_id) - except exc.UCloudException as uerr: - err_msg = 'ucloud error: {ucloud_error}'.format(ucloud_error=uerr) - except Exception as e: - err_msg = 'error: {}'.format(e) - - else: - err_msg = 'In Development.' - - return err_msg - - @run_on_executor(executor='_thread_pool') - def redis_api(self, account, region, access_id, access_key, project_id): - """ - 测试redis API权限 - :return: - """ - # 错误收集 - err_msg = '' - - if account == 'AWS': - ins_log.read_log('info', 'AWS redis API TEST') - # err_msg = 'In Development' - try: - AwsCacheApi.permission_auth(access_id, access_key, region) - except Exception as e: - err_msg = 'Error:{}'.format(e) - - elif account == '阿里云': - ins_log.read_log('info', 'Aliyun Redis API TEST') - try: - AliyunRedisApi.permission_auth(access_id, access_key, region) - except Exception as e: - err_msg = 'Error:{}'.format(e) - - elif account == '腾讯云': - ins_log.read_log('info', 'QCloud Redis API TEST') - try: - obj = RedisApi(access_id, access_key, region) - result_data = obj.test_auth() - if result_data['Response'].get('Error'): - err_msg = 'Error：{}'.format(result_data['Response']) - except Exception as e: - ins_log.read_log('Error', e) - elif account == 'UCloud': - ins_log.read_log('info', 'UCloud Uredis API TEST') - try: - UredisAPI.permission_auth(access_id=access_id, access_key=access_key, region=region, - project_id=project_id) - except exc.UCloudException as uerr: - err_msg = 'ucloud error: {ucloud_error}'.format(ucloud_error=uerr) - except Exception as e: - err_msg = 'error: {}'.format(e) - else: - # 其余厂商 - err_msg = 'In Development' - return err_msg - - @gen.coroutine - def post(self, *args, **kwargs): - data = json.loads(self.request.body.decode("utf-8")) - api_type = data.get('api_type', None) - account = data.get('account', None) - region = data.get('region', None) - access_id = data.get('access_id', None) - access_key = data.get('access_key', None) - project_id = data.get('project_id', None) - huawei_cloud = data.get('huawei_cloud', None) - huawei_instance_id = data.get('huawei_instance_id', None) - - if not account or not region or not access_id or not access_key: - return self.write(dict(code=-2, msg="测试必须要包含：厂商、Region、 Access_id 、Access_key")) - - if account == '华为云': - if not project_id or not huawei_cloud or not huawei_instance_id: - return self.write(dict(code=-2, msg="华为云测试必须包含：Cloud地址、区域项目ID、一个实例ID")) - elif account == 'UCloud': - if not project_id: - return self.write(dict(code=-2, msg="UCloud测试必须包含：项目ID")) - - if api_type == 'server': - error_msg = yield self.server_api(account, region, access_id, access_key, project_id, huawei_cloud, - huawei_instance_id) - if error_msg: return self.write(dict(code=-1, msg=error_msg)) - return self.write(dict(code=0, msg='Successful')) - elif api_type == 'rds': - error_msg = yield self.rds_api(account, region, access_id, access_key, project_id) - if error_msg: return self.write(dict(code=-1, msg=error_msg)) - return self.write(dict(code=0, msg='Successful')) - elif api_type == 'redis': - error_msg = yield self.redis_api(account, region, access_id, access_key, project_id) - if error_msg: return self.write(dict(code=-1, msg=error_msg)) - return self.write(dict(code=0, msg='Successful')) - else: - return self.write(dict(code=-1, msg='In Development')) - - -class HanderUpdateOSServer(tornado.web.RequestHandler): - '''前端手动触发从云厂商更新资产,使用异步方法''' - _thread_pool = ThreadPoolExecutor(3) - - @run_on_executor(executor='_thread_pool') - def handler_update_task(self): - aliyun_ecs_update() - time.sleep(2) - aws_ec2_update() - time.sleep(2) - qcloud_cvm_update() - time.sleep(2) - huawei_ecs_update() - time.sleep(2) - ucloud_uhost_update() - - @gen.coroutine - def get(self, *args, **kwargs): - yield self.handler_update_task() - return self.write(dict(code=0, msg='服务器信息拉取完成')) - - -class HanderUpdateOSRds(tornado.web.RequestHandler): - '''前端手动触发从云厂商更新资产,使用异步方法''' - _thread_pool = ThreadPoolExecutor(3) - - @run_on_executor(executor='_thread_pool') - def handler_update_task(self): - aws_rds_update() - time.sleep(1) - qcloud_cdb_update() - time.sleep(1) - aliyun_rds_update() - time.sleep(1) - ucloud_udb_update() - - @gen.coroutine - def get(self, *args, **kwargs): - yield self.handler_update_task() - return self.write(dict(code=0, msg='数据库信息拉取完成')) - - -class HanderUpdateOSRedis(tornado.web.RequestHandler): - '''前端手动触发从云厂商更新资产,使用异步方法''' - _thread_pool = ThreadPoolExecutor(3) - - @run_on_executor(executor='_thread_pool') - def handler_update_task(self): - qcloud_redis_update() - time.sleep(1) - aliyun_redis_update() - time.sleep(1) - aws_cache_update() - time.sleep(1) - ucloud_uredis_update() - - @gen.coroutine - def get(self, *args, **kwargs): - yield self.handler_update_task() - return self.write(dict(code=0, msg='Redis信息拉取完成')) - - -asset_configs_urls = [ - (r"/v1/cmdb/asset_configs/", AssetConfigsHandler), - (r"/v1/cmdb/asset_configs/handler_update_server/", HanderUpdateOSServer), - (r"/v1/cmdb/asset_configs/handler_update_rds/", HanderUpdateOSRds), - (r"/v1/cmdb/asset_configs/handler_update_redis/", HanderUpdateOSRedis), - (r"/v1/cmdb/asset_configs/api_permission/", ApiPermissionHandler), -] +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Time : 2019/5/14 10:30 +# @Author : Fred Yangxiaofei +# @File : asset_configs_handler.py +# @Role : 资产配置，主要用来实现AWS/aliyun/qcloud 多账户云机器自动录入CMDB + +import json +import time +import openstack +import keystoneauth1 +import tornado.web +from tornado import gen +from concurrent.futures import ThreadPoolExecutor +from tornado.concurrent import run_on_executor + +from models.server import AssetConfigs, model_to_dict +from websdk.db_context import DBContext +from websdk.web_logs import ins_log +from opssdk.operate import MyCryptV2 +from libs.base_handler import BaseHandler +from libs.aws.ec2 import Ec2Api +from libs.aws.rds import RDSApi as AwsRdsApi +from libs.aws.ec2 import main as aws_ec2_update +from libs.aws.rds import main as aws_rds_update +from libs.aws.elasticache import main as aws_cache_update +from libs.aws.elasticache import CacheApi as AwsCacheApi +from libs.aliyun.ecs import EcsAPi +from libs.aliyun.rds import RdsApi as AliyunRdsApi +from libs.aliyun.redis import RedisApi as AliyunRedisApi +from libs.aliyun.ecs import main as aliyun_ecs_update +from libs.aliyun.rds import main as aliyun_rds_update +from libs.aliyun.redis import main as aliyun_redis_update +from libs.qcloud.cvm import CVMApi +from libs.qcloud.cdb import CDBApi as QcloudCdbApi +from libs.qcloud.redis import RedisApi +from libs.qcloud.cvm import main as qcloud_cvm_update +from libs.qcloud.cdb import main as qcloud_cdb_update +from libs.qcloud.redis import main as qcloud_redis_update +from libs.huaweiyun.huawei_ecs import HuaweiEcsApi +from libs.huaweiyun.huawei_ecs import main as huawei_ecs_update +from ucloud.core import exc +from libs.ucloud.uhost import UHostAPI +from libs.ucloud.uhost import main as ucloud_uhost_update +from libs.ucloud.udb import UdbAPI +from libs.ucloud.udb import main as ucloud_udb_update +from libs.ucloud.uredis import UredisAPI +from libs.ucloud.uredis import main as ucloud_uredis_update + +mc = MyCryptV2() # 实例化 + + +class AssetConfigsHandler(BaseHandler): + def get(self, *args, **kwargs): + key = self.get_argument('key', default=None, strip=True) + value = self.get_argument('value', default=None, strip=True) + asset_configs_list = [] + with DBContext('r') as session: + if key and value: + asset_configs_data = session.query(AssetConfigs).filter_by(**{key: value}).all() + else: + asset_configs_data = session.query(AssetConfigs).all() + + for data in asset_configs_data: + data_dict = model_to_dict(data) + data_dict['create_time'] = str(data_dict['create_time']) + data_dict['update_time'] = str(data_dict['update_time']) + # 解密AccessKey + if data_dict.get('access_key'): + data_dict['access_key'] = mc.my_decrypt(data_dict.get('access_key')) + asset_configs_list.append(data_dict) + + return self.write(dict(code=0, msg='获取成功', data=asset_configs_list)) + + def post(self, *args, **kwargs): + data = json.loads(self.request.body.decode("utf-8")) + name = data.get('name', None) + account = data.get('account', None) + region = data.get('region', None).strip() + access_id = data.get('access_id', None).strip() + access_key = data.get('access_key', None).strip() + default_admin_user = data.get('default_admin_user', None) + state = data.get('state', None) + remarks = data.get('remarks', None) + # 华为云需要额外三个数据 + project_id = data.get('project_id', 'Null') + huawei_cloud = data.get('huawei_cloud', 'Null') + huawei_instance_id = data.get('huawei_instance_id', 'Null') + + if not name or not account or not region or not access_id or not access_key or not state: + return self.write(dict(code=-2, msg='关键参数不能为空')) + + with DBContext('r', None, True) as session: + exist_id = session.query(AssetConfigs.id).filter(AssetConfigs.name == name).first() + + if exist_id: + return self.write(dict(code=-2, msg='不要重复记录')) + + # 对密钥进行加密再写数据库 + _access_key = mc.my_encrypt(access_key) + + with DBContext('w', None, True) as session: + new_asset_config = AssetConfigs(name=name, account=account, region=region, access_id=access_id, + access_key=_access_key, default_admin_user=default_admin_user, state=state, + remarks=remarks, project_id=project_id, huawei_cloud=huawei_cloud, + huawei_instance_id=huawei_instance_id) + session.add(new_asset_config) + + return self.write(dict(code=0, msg='添加成功')) + + def put(self, *args, **kwargs): + data = json.loads(self.request.body.decode("utf-8")) + id = data.get('id', None) + name = data.get('name', None) + account = data.get('account', None) + region = data.get('region', None) + access_id = data.get('access_id', None) + access_key = data.get('access_key', None) + default_admin_user = data.get('default_admin_user', None) + state = data.get('state', None) + remarks = data.get('remarks', None) + # 华为云需要额外三个数据 + project_id = data.get('project_id', None) + huawei_cloud = data.get('huawei_cloud', None) + huawei_instance_id = data.get('huawei_instance_id', None) + + if not name or not account or not region or not access_id or not access_key or not state: + return self.write(dict(code=-2, msg='关键参数不能为空')) + + # 对密钥进行加密再写数据库 + _access_key = mc.my_encrypt(access_key) + with DBContext('w', None, True) as session: + session.query(AssetConfigs).filter(AssetConfigs.id == id).update( + {AssetConfigs.account: account, AssetConfigs.access_id: access_id, AssetConfigs.access_key: _access_key, + AssetConfigs.default_admin_user: default_admin_user, AssetConfigs.state: state, + AssetConfigs.remarks: remarks, AssetConfigs.project_id: project_id, + AssetConfigs.huawei_cloud: huawei_cloud, AssetConfigs.huawei_instance_id: huawei_instance_id}) + return self.write(dict(code=0, msg='编辑成功')) + + def patch(self, *args, **kwargs): + '''开关控制，开启/禁用''' + data = json.loads(self.request.body.decode("utf-8")) + id = data.get('id') + + msg = 'Not Fount!' + + if not id: + return self.write(dict(code=-2, msg='关键参数不能为空')) + + with DBContext('r') as session: + config_state = session.query(AssetConfigs.state).filter(AssetConfigs.id == id).first() + + if not config_state: + return self.write(dict(code=0, msg=msg)) + + if config_state[0] == 'false': + msg = '启用成功' + new_state = 'true' + + elif config_state[0] == 'true': + msg = '禁用成功' + new_state = 'false' + + with DBContext('w', None, True) as session: + # print(new_state) + session.query(AssetConfigs).filter(AssetConfigs.id == id).update({AssetConfigs.state: new_state}) + + return self.write(dict(code=0, msg=msg)) + + def delete(self, *args, **kwargs): + data = json.loads(self.request.body.decode("utf-8")) + id = data.get('id') + if not id: + return self.write(dict(code=-2, msg='关键参数不能为空')) + + with DBContext('w', None, True) as session: + session.query(AssetConfigs).filter(AssetConfigs.id == id).delete(synchronize_session=False) + + self.write(dict(code=0, msg='删除成功')) + + +class ApiPermissionHandler(BaseHandler): + '''测试用户填写的信息及认证是否正确,防止主进程卡死，使用异步方法测试''' + _thread_pool = ThreadPoolExecutor(3) + + @run_on_executor(executor='_thread_pool') + def server_api(self, account, region, access_id, access_key, project_id, huawei_cloud, huawei_instance_id): + """ + 测试各厂商server API的权限 + :return: + """ + # 收集测试错误日志 + err_msg = '' + # 测试接口的时候默认管理用户不需要 + default_admin_user = '' + if account == 'AWS': + ins_log.read_log('info', 'AWS EC2 API TEST') + obj = Ec2Api(access_id, access_key, region, default_admin_user) + try: + obj.test_auth() + except Exception as e: + err_msg = 'Error：{}'.format(e) + elif account == '阿里云': + ins_log.read_log('info', 'Aliyun ECS API TEST') + obj = EcsAPi(access_id, access_key, region, default_admin_user) + try: + obj.test_auth() + except Exception as e: + err_msg = 'Error：{}'.format(e) + + elif account == '腾讯云': + ins_log.read_log('info', 'Qcloud CVM API TEST') + obj = CVMApi(access_id, access_key, region, default_admin_user) + try: + result_data = obj.test_auth() + if result_data['Response'].get('Error'): + err_msg = 'Error：{}'.format(result_data['Response']) + except Exception as e: + ins_log.read_log('error', e) + + elif account == '华为云': + ins_log.read_log('info', 'HuaweiCloud ECS API TEST') + obj = HuaweiEcsApi(access_id=access_id, access_key=access_key, region=region, cloud=huawei_cloud, + project_id=project_id, + default_admin_user=default_admin_user) + try: + obj.test_auth(huawei_instance_id) + except keystoneauth1.exceptions.http.Unauthorized: + err_msg = '请检查AccessID和AccessKey权限是否正确' + except openstack.exceptions.HttpException as err: + err_msg = 'openstack error {}'.format(err) + except Exception as e: + err_msg = 'error: {}'.format(e) + elif account == 'UCloud': + ins_log.read_log('info', 'UCloud Uhost API TEST') + try: + UHostAPI.permission_auth(access_id=access_id, access_key=access_key, region=region, + project_id=project_id) + except exc.UCloudException as uerr: + err_msg = 'ucloud error: {ucloud_error}'.format(ucloud_error=uerr) + except Exception as e: + err_msg = 'error: {}'.format(e) + else: + err_msg = 'In Development.' + return err_msg + + @run_on_executor(executor='_thread_pool') + def rds_api(self, account, region, access_id, access_key, project_id): + """ + 测试DB API权限 + :return: + """ + # 错误收集 + err_msg = '' + if account == 'AWS': + ins_log.read_log('info', 'AWS RDS API TEST') + obj = AwsRdsApi(access_id, access_key, region, ) + try: + obj.test_auth() + except Exception as e: + err_msg = 'Error：{}'.format(e) + elif account == '阿里云': + ins_log.read_log('info', 'Aliyun RDS API TEST') + obj = AliyunRdsApi(access_id, access_key, region) + try: + obj.test_auth() + except Exception as e: + err_msg = 'Error：{}'.format(e) + + elif account == '腾讯云': + ins_log.read_log('info', 'Qcloud CDB API TEST') + obj = QcloudCdbApi(access_id, access_key, region) + try: + result_data = obj.test_auth() + if result_data['Response'].get('Error'): + err_msg = 'Error：{}'.format(result_data['Response']) + except Exception as e: + ins_log.read_log('error', e) + elif account == 'UCloud': + ins_log.read_log('info', 'UCloud UDB API TEST') + try: + UdbAPI.permission_auth(access_id=access_id, access_key=access_key, region=region, + project_id=project_id) + except exc.UCloudException as uerr: + err_msg = 'ucloud error: {ucloud_error}'.format(ucloud_error=uerr) + except Exception as e: + err_msg = 'error: {}'.format(e) + + else: + err_msg = 'In Development.' + + return err_msg + + @run_on_executor(executor='_thread_pool') + def redis_api(self, account, region, access_id, access_key, project_id): + """ + 测试redis API权限 + :return: + """ + # 错误收集 + err_msg = '' + + if account == 'AWS': + ins_log.read_log('info', 'AWS redis API TEST') + # err_msg = 'In Development' + try: + AwsCacheApi.permission_auth(access_id, access_key, region) + except Exception as e: + err_msg = 'Error:{}'.format(e) + + elif account == '阿里云': + ins_log.read_log('info', 'Aliyun Redis API TEST') + try: + AliyunRedisApi.permission_auth(access_id, access_key, region) + except Exception as e: + err_msg = 'Error:{}'.format(e) + + elif account == '腾讯云': + ins_log.read_log('info', 'QCloud Redis API TEST') + try: + obj = RedisApi(access_id, access_key, region) + result_data = obj.test_auth() + if result_data['Response'].get('Error'): + err_msg = 'Error：{}'.format(result_data['Response']) + except Exception as e: + ins_log.read_log('Error', e) + elif account == 'UCloud': + ins_log.read_log('info', 'UCloud Uredis API TEST') + try: + UredisAPI.permission_auth(access_id=access_id, access_key=access_key, region=region, + project_id=project_id) + except exc.UCloudException as uerr: + err_msg = 'ucloud error: {ucloud_error}'.format(ucloud_error=uerr) + except Exception as e: + err_msg = 'error: {}'.format(e) + else: + # 其余厂商 + err_msg = 'In Development' + return err_msg + + @gen.coroutine + def post(self, *args, **kwargs): + data = json.loads(self.request.body.decode("utf-8")) + api_type = data.get('api_type', None) + account = data.get('account', None) + region = data.get('region', None) + access_id = data.get('access_id', None) + access_key = data.get('access_key', None) + project_id = data.get('project_id', None) + huawei_cloud = data.get('huawei_cloud', None) + huawei_instance_id = data.get('huawei_instance_id', None) + + if not account or not region or not access_id or not access_key: + return self.write(dict(code=-2, msg="测试必须要包含：厂商、Region、 Access_id 、Access_key")) + + if account == '华为云': + if not project_id or not huawei_cloud or not huawei_instance_id: + return self.write(dict(code=-2, msg="华为云测试必须包含：Cloud地址、区域项目ID、一个实例ID")) + elif account == 'UCloud': + if not project_id: + return self.write(dict(code=-2, msg="UCloud测试必须包含：项目ID")) + + if api_type == 'server': + error_msg = yield self.server_api(account, region, access_id, access_key, project_id, huawei_cloud, + huawei_instance_id) + if error_msg: return self.write(dict(code=-1, msg=error_msg)) + return self.write(dict(code=0, msg='Successful')) + elif api_type == 'rds': + error_msg = yield self.rds_api(account, region, access_id, access_key, project_id) + if error_msg: return self.write(dict(code=-1, msg=error_msg)) + return self.write(dict(code=0, msg='Successful')) + elif api_type == 'redis': + error_msg = yield self.redis_api(account, region, access_id, access_key, project_id) + if error_msg: return self.write(dict(code=-1, msg=error_msg)) + return self.write(dict(code=0, msg='Successful')) + else: + return self.write(dict(code=-1, msg='In Development')) + + +class HanderUpdateOSServer(tornado.web.RequestHandler): + '''前端手动触发从云厂商更新资产,使用异步方法''' + _thread_pool = ThreadPoolExecutor(3) + + @run_on_executor(executor='_thread_pool') + def handler_update_task(self): + aliyun_ecs_update() + time.sleep(2) + aws_ec2_update() + time.sleep(2) + qcloud_cvm_update() + time.sleep(2) + huawei_ecs_update() + time.sleep(2) + ucloud_uhost_update() + + @gen.coroutine + def get(self, *args, **kwargs): + yield self.handler_update_task() + return self.write(dict(code=0, msg='服务器信息拉取完成')) + + +class HanderUpdateOSRds(tornado.web.RequestHandler): + '''前端手动触发从云厂商更新资产,使用异步方法''' + _thread_pool = ThreadPoolExecutor(3) + + @run_on_executor(executor='_thread_pool') + def handler_update_task(self): + aws_rds_update() + time.sleep(1) + qcloud_cdb_update() + time.sleep(1) + aliyun_rds_update() + time.sleep(1) + ucloud_udb_update() + + @gen.coroutine + def get(self, *args, **kwargs): + yield self.handler_update_task() + return self.write(dict(code=0, msg='数据库信息拉取完成')) + + +class HanderUpdateOSRedis(tornado.web.RequestHandler): + '''前端手动触发从云厂商更新资产,使用异步方法''' + _thread_pool = ThreadPoolExecutor(3) + + @run_on_executor(executor='_thread_pool') + def handler_update_task(self): + qcloud_redis_update() + time.sleep(1) + aliyun_redis_update() + time.sleep(1) + aws_cache_update() + time.sleep(1) + ucloud_uredis_update() + + @gen.coroutine + def get(self, *args, **kwargs): + yield self.handler_update_task() + return self.write(dict(code=0, msg='Redis信息拉取完成')) + + +asset_configs_urls = [ + (r"/v1/cmdb/asset_configs/", AssetConfigsHandler), + (r"/v1/cmdb/asset_configs/handler_update_server/", HanderUpdateOSServer), + (r"/v1/cmdb/asset_configs/handler_update_rds/", HanderUpdateOSRds), + (r"/v1/cmdb/asset_configs/handler_update_redis/", HanderUpdateOSRedis), + (r"/v1/cmdb/asset_configs/api_permission/", ApiPermissionHandler), +] diff --git a/biz/handlers/asset_db_handler.py b/biz/handlers/asset_db_handler.py index 7e490ce..4010ca8 100644 --- a/biz/handlers/asset_db_handler.py +++ b/biz/handlers/asset_db_handler.py @@ -1,386 +1,386 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# @Time : 2019/4/21 8:37 -# @Author : Fred Yangxiaofei -# @File : asset_db_handler.py -# @Role : DB - -import json -from sqlalchemy import or_ -from libs.base_handler import BaseHandler -from models.db import DBTag, DB, model_to_dict -from models.server import Tag, AssetOperationalAudit -from websdk.db_context import DBContext -from websdk.web_logs import ins_log - - -class DBHandler(BaseHandler): - def get(self, *args, **kwargs): - nickname = self.get_current_nickname() - key = self.get_argument('key', default=None, strip=True) - value = self.get_argument('value', default=None, strip=True) - page_size = self.get_argument('page', default=1, strip=True) - limit = self.get_argument('limit', default='888', strip=True) - limit_start = (int(page_size) - 1) * int(limit) - db_list = [] - - with DBContext('r') as session: - ### 通过TAG搜索 - if key == 'tag_name' and value: - count = session.query(DB).outerjoin(DBTag, DB.id == DBTag.db_id).outerjoin(Tag, Tag.id == - DBTag.tag_id).filter( - Tag.tag_name == value).count() - db_info = session.query(DB).outerjoin(DBTag, DB.id == DBTag.db_id).outerjoin(Tag, Tag.id == - DBTag.tag_id).filter( - Tag.tag_name == value).order_by(DB.id).offset(limit_start).limit(int(limit)) - for msg in db_info: - tag_list = [] - data_dict = model_to_dict(msg) - data_dict['create_time'] = str(data_dict['create_time']) - data_dict['update_time'] = str(data_dict['update_time']) - db_tags = session.query(Tag.tag_name).outerjoin(DBTag, Tag.id == DBTag.tag_id).filter( - DBTag.db_id == data_dict['id']).all() - for t in db_tags: - tag_list.append(t[0]) - data_dict['tag_list'] = tag_list - db_list.append(data_dict) - return self.write(dict(code=0, msg='获取成功', count=count, data=db_list)) - - ### 监听搜索 - if key and key != 'tag_name' and not value: - # TODO 超管查所有 - if self.is_superuser: - count = session.query(DB).filter(or_(DB.db_code.like('%{}%'.format(key)), - DB.db_host.like('%{}%'.format(key)), - DB.idc.like('%{}%'.format(key)), - DB.db_user.like('%{}%'.format(key)), - DB.db_pwd.like('%{}%'.format(key)), - DB.proxy_host.like('%{}%'.format(key)), - DB.db_type.like('%{}%'.format(key)), - DB.db_version.like('%{}%'.format(key)), - DB.db_mark.like('%{}%'.format(key)), - DB.state.like('%{}%'.format(key)), - DB.db_env.like('%{}%'.format(key)))).count() - - db_info = session.query(DB).filter(or_(DB.db_code.like('%{}%'.format(key)), - DB.db_host.like('%{}%'.format(key)), - DB.db_user.like('%{}%'.format(key)), - DB.db_pwd.like('%{}%'.format(key)), - DB.proxy_host.like('%{}%'.format(key)), - DB.db_type.like('%{}%'.format(key)), - DB.db_version.like('%{}%'.format(key)), - DB.db_mark.like('%{}%'.format(key)), - DB.state.like('%{}%'.format(key)), - DB.idc.like('%{}%'.format(key)), - DB.db_env.like('%{}%'.format(key)))).order_by(DB.id) - - else: - # TODO 普通用户只能看到有权限的 - db_id_list = [] - with DBContext('r') as session: - the_dbs = session.query(DBTag.db_id).filter(DBTag.tag_id.in_( - session.query(Tag.id).filter(or_(Tag.users.like('%{}%'.format(nickname)))))) - for s in the_dbs: - db_id_list.append(s[0]) - # 去重下列表,万一有重复的呢 - set_db_id_list = set(db_id_list) - count = session.query(DB).filter(DB.id.in_(set_db_id_list)).filter( - or_(DB.db_code.like('%{}%'.format(key)), - DB.db_host.like('%{}%'.format(key)), - DB.db_user.like('%{}%'.format(key)), - DB.db_pwd.like('%{}%'.format(key)), - DB.proxy_host.like('%{}%'.format(key)), - DB.db_type.like('%{}%'.format(key)), - DB.db_version.like('%{}%'.format(key)), - DB.db_mark.like('%{}%'.format(key)), - DB.state.like('%{}%'.format(key)), - DB.idc.like('%{}%'.format(key)), - DB.db_env.like('%{}%'.format(key)))).count() - - db_info = session.query(DB).filter(DB.id.in_(set_db_id_list)).filter( - or_(DB.db_code.like('%{}%'.format(key)), - DB.db_host.like('%{}%'.format(key)), - DB.db_user.like('%{}%'.format(key)), - DB.db_pwd.like('%{}%'.format(key)), - DB.proxy_host.like('%{}%'.format(key)), - DB.db_type.like('%{}%'.format(key)), - DB.db_version.like('%{}%'.format(key)), - DB.db_mark.like('%{}%'.format(key)), - DB.state.like('%{}%'.format(key)), - DB.idc.like('%{}%'.format(key)), - DB.db_env.like('%{}%'.format(key)))).order_by(DB.id).offset( - limit_start).limit(int(limit)) - - for msg in db_info: - tag_list = [] - data_dict = model_to_dict(msg) - data_dict['create_time'] = str(data_dict['create_time']) - data_dict['update_time'] = str(data_dict['update_time']) - db_tags = session.query(Tag.tag_name).outerjoin(DBTag, Tag.id == DBTag.tag_id).filter( - DBTag.db_id == data_dict['id']).all() - for t in db_tags: - tag_list.append(t[0]) - data_dict['tag_list'] = tag_list - db_list.append(data_dict) - - return self.write(dict(code=0, msg='获取成功', count=count, data=db_list)) - - ### 888查看所有的数据库 - if limit == '888': - count = session.query(DB).count() - db_info = session.query(DB).order_by(DB.id).all() - else: - # TODO 超管查所有 - if self.is_superuser: - if key and value: - count = session.query(DB).filter_by(**{key: value}).count() - db_info = session.query(DB).filter_by(**{key: value}).order_by(DB.id).offset(limit_start).limit( - int(limit)) - else: - count = session.query(DB).count() - db_info = session.query(DB).order_by(DB.id).offset(limit_start).limit(int(limit)) - else: - # TODO 普通用户只给有权限的DB,根据用户查Tagid, 根据Tagid查询出来关联的DBID，根据DBID返回主机详情 - db_id_list = [] - with DBContext('r') as session: - # 子查询查出来server_id - the_dbs = session.query(DBTag.db_id).filter(DBTag.tag_id.in_( - session.query(Tag.id).filter(or_(Tag.users.like('%{}%'.format(nickname)))))) - for d in the_dbs: - db_id_list.append(d[0]) - # 去重下列表,万一有重复的呢 - set_db_id_list = set(db_id_list) - if key and value: - # 根据Keyvalue获取 - count = session.query(DB).filter(DB.id.in_(set_db_id_list)).filter_by( - **{key: value}).count() - db_info = session.query(DB).filter(DB.id.in_(set_db_id_list)).filter_by( - **{key: value}).order_by(DB.id).offset( - limit_start).limit(int(limit)) - else: - # 获取主机详情 - - count = session.query(DB).filter(DB.id.in_(set_db_id_list)).count() - db_info = session.query(DB).filter(DB.id.in_(set_db_id_list)).offset( - limit_start).limit(int(limit)) - - for msg in db_info: - tag_list = [] - data_dict = model_to_dict(msg) - db_tags = session.query(Tag.tag_name).outerjoin(DBTag, Tag.id == DBTag.tag_id).filter( - DBTag.db_id == data_dict['id']).all() - for t in db_tags: - tag_list.append(t[0]) - - data_dict['create_time'] = str(data_dict['create_time']) - data_dict['update_time'] = str(data_dict['update_time']) - data_dict['tag_list'] = tag_list - db_list.append(data_dict) - - self.write(dict(code=0, msg='获取成功', count=count, data=db_list)) - - def post(self, *args, **kwargs): - data = json.loads(self.request.body.decode("utf-8")) - nickname = self.get_current_nickname() - db_code = data.get('db_code', None) - db_host = data.get('db_host', None) - db_port = data.get('db_port', 3306) - db_user = data.get('db_user', None) - db_pwd = data.get('db_pwd', None) - db_env = data.get('db_env', None) - proxy_host = data.get('proxy_host', None) - db_type = data.get('db_type', 'mysql') - db_version = data.get('db_version', None) - db_mark = data.get('db_mark', '写') - tag_list = data.get('tag_list', []) - db_detail = data.get('db_detail', None) - if not db_code or not db_host or not db_port or not db_user: - return self.write(dict(code=-1, msg='关键参数不能为空')) - - with DBContext('r') as session: - exist_id = session.query(DB.id).filter(DB.db_code == db_code, DB.db_host == db_host, DB.db_port == db_port, - DB.db_user == db_user, DB.db_env == db_env, - DB.proxy_host == proxy_host, DB.db_type == db_type, - db_version == db_version, - DB.db_mark == db_mark).first() - if exist_id: - return self.write(dict(code=-2, msg='不要重复记录')) - - with DBContext('w', None, True) as session: - new_db = DB(db_code=db_code, db_host=db_host, db_port=db_port, db_user=db_user, db_pwd=db_pwd, - db_env=db_env, proxy_host=proxy_host, db_type=db_type, db_version=db_version, db_mark=db_mark, - db_detail=db_detail) - session.add(new_db) - - all_tags = session.query(Tag.id).filter(Tag.tag_name.in_(tag_list)).order_by(Tag.id).all() - if all_tags: - for tag_id in all_tags: - session.add(DBTag(db_id=new_db.id, tag_id=tag_id[0])) - # 记录,记录错误也不要影响用户正常添加 - try: - with DBContext('w', None, True) as session: - - new_record = AssetOperationalAudit(username=nickname, request_object='数据库',request_host=db_host, - request_method='新增', modify_data=data) - session.add(new_record) - except Exception as err: - ins_log.read_log('error', 'operational_audit error:{err}'.format(err=err)) - - return self.write(dict(code=0, msg='添加成功')) - - def put(self, *args, **kwargs): - data = json.loads(self.request.body.decode("utf-8")) - nickname = self.get_current_nickname() - db_id = data.get('id', None) - db_code = data.get('db_code', None) - db_host = data.get('db_host', None) - db_port = data.get('db_port', 3306) - db_user = data.get('db_user', None) - db_pwd = data.get('db_pwd', None) - db_env = data.get('db_env', None) - proxy_host = data.get('proxy_host', None) - db_type = data.get('db_type', 'mysql') - db_version = data.get('db_version', None) - db_mark = data.get('db_mark', '写') - idc = data.get('idc', None) - tag_list = data.get('tag_list', []) - db_detail = data.get('db_detail', None) - if not db_id or not db_code or not db_host or not db_port or not db_user: - return self.write(dict(code=-1, msg='关键参数不能为空')) - - with DBContext('w', None, True) as session: - all_tags = session.query(Tag.id).filter(Tag.tag_name.in_(tag_list)).order_by(Tag.id).all() - session.query(DBTag).filter(DBTag.db_id == db_id).delete(synchronize_session=False) - if all_tags: - for tag_id in all_tags: - session.add(DBTag(db_id=int(db_id), tag_id=tag_id[0])) - - session.query(DB).filter(DB.id == int(db_id)).update({DB.db_code: db_code, DB.db_host: db_host, - DB.db_port: db_port, DB.db_user: db_user, - DB.db_pwd: db_pwd, DB.db_env: db_env, - DB.proxy_host: proxy_host, DB.db_type: db_type, - DB.db_version: db_version, DB.idc: idc, - DB.db_mark: db_mark, DB.db_detail: db_detail}) - # 记录操作,不成功直接Pass - try: - modify_data = data - with DBContext('w', None, True) as session: - data_info = session.query(DB).filter(DB.id == int(db_id)).all() - for data in data_info: - origion_data = model_to_dict(data) - origion_data['create_time'] = str(origion_data['create_time']) - origion_data['update_time'] = str(origion_data['update_time']) - new_record = AssetOperationalAudit(username=nickname, request_object='数据库',request_host=db_host, - request_method='更新', original_data=origion_data, - modify_data=modify_data) - session.add(new_record) - except Exception as err: - ins_log.read_log('error', 'operational_audit error:{err}'.format(err=err)) - - return self.write(dict(code=0, msg='编辑成功')) - - def delete(self, *args, **kwargs): - data = json.loads(self.request.body.decode("utf-8")) - nickname = self.get_current_nickname() - db_id = data.get('db_id', None) - id_list = data.get('id_list', None) - - - # 记录操作,不成功直接Pass - try: - with DBContext('w', None, True) as session: - if db_id: - data_info = session.query(DB).filter(DB.id == int(db_id)).all() - for data in data_info: - origion_data = model_to_dict(data) - origion_data['create_time'] = str(origion_data['create_time']) - origion_data['update_time'] = str(origion_data['update_time']) - new_record = AssetOperationalAudit(username=nickname, request_object='数据库',request_host=origion_data.get('db_host'), - request_method='删除', original_data=origion_data) - session.add(new_record) - elif id_list: - for i in id_list: - data_info = session.query(DB).filter(DB.id == i).all() - for data in data_info: - origion_data = model_to_dict(data) - origion_data['create_time'] = str(origion_data['create_time']) - origion_data['update_time'] = str(origion_data['update_time']) - new_record = AssetOperationalAudit(username=nickname, request_object='数据库',request_host=origion_data.get('db_host'), - request_method='删除', original_data=origion_data) - session.add(new_record) - except Exception as err: - ins_log.read_log('error', 'operational_audit error:{err}'.format(err=err)) - - with DBContext('w', None, True) as session: - if db_id: - session.query(DB).filter(DB.id == int(db_id)).delete(synchronize_session=False) - session.query(DBTag).filter(DBTag.db_id == int(db_id)).delete(synchronize_session=False) - elif id_list: - for i in id_list: - session.query(DB).filter(DB.id == i).delete(synchronize_session=False) - session.query(DBTag).filter(DBTag.db_id == i).delete(synchronize_session=False) - else: - return self.write(dict(code=1, msg='关键参数不能为空')) - return self.write(dict(code=0, msg='删除成功')) - - -class MultiAddDBHandler(BaseHandler): - def post(self, *args, **kwargs): - data = json.loads(self.request.body.decode("utf-8")) - nickname = self.get_current_nickname() - - # 列表包str格式，str空格切割 - if not data: - return self.write(dict(code=-1, msg='不能为空')) - - #记录下操作，即使报错也不影响程序 - try: - with DBContext('w', None, True) as session: - - new_record = AssetOperationalAudit(username=nickname, request_object='数据库', - request_method='批量添加', modify_data=data) - session.add(new_record) - except Exception as err: - ins_log.read_log('error', 'operational_audit error:{err}'.format(err=err)) - - # 判断下格式长度 - with DBContext('w', None, True) as session: - for i in data: - if i: - db_info = i.split(' ') - ins_log.read_log('info', 'MultiDB:{db_info}'.format(db_info=db_info)) - if len(db_info) != 6: - return self.write(dict(code=-2, msg='格式错误')) - - db_type = db_info[0] - db_name = db_info[1] - db_host = db_info[2] - db_port = db_info[3] - db_user = db_info[4] - db_passwd = db_info[5] - - if not type(db_port) is int and int(db_port) >= 65535: - return self.write(dict(code=-1, msg="端口格式不正确")) - - exist_id = session.query(DB.id).filter(DB.db_code == db_name).first() - exist_ip = session.query(DB.id).filter(DB.db_host == db_host).first() - if exist_id or exist_ip: - return self.write( - dict(code=-2, - msg='IP:{address}/Hostname:{hostname}已经存在,不要重复记录'.format(address=db_host, - hostname=db_name))) - - new_db = DB(db_code=db_name, db_host=db_host, db_port=db_port, db_user=db_user, db_type=db_type, - db_pwd=db_passwd, state='Handle') - session.add(new_db) - session.commit() - - return self.write(dict(code=0, msg='批量添加成功')) - - -asset_db_urls = [ - (r"/v1/cmdb/db/", DBHandler), - (r"/v1/cmdb/db/multi_add/", MultiAddDBHandler), -] -if __name__ == "__main__": - pass +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Time : 2019/4/21 8:37 +# @Author : Fred Yangxiaofei +# @File : asset_db_handler.py +# @Role : DB + +import json +from sqlalchemy import or_ +from libs.base_handler import BaseHandler +from models.db import DBTag, DB, model_to_dict +from models.server import Tag, AssetOperationalAudit +from websdk.db_context import DBContext +from websdk.web_logs import ins_log + + +class DBHandler(BaseHandler): + def get(self, *args, **kwargs): + nickname = self.get_current_nickname() + key = self.get_argument('key', default=None, strip=True) + value = self.get_argument('value', default=None, strip=True) + page_size = self.get_argument('page', default=1, strip=True) + limit = self.get_argument('limit', default='888', strip=True) + limit_start = (int(page_size) - 1) * int(limit) + db_list = [] + + with DBContext('r') as session: + ### 通过TAG搜索 + if key == 'tag_name' and value: + count = session.query(DB).outerjoin(DBTag, DB.id == DBTag.db_id).outerjoin(Tag, Tag.id == + DBTag.tag_id).filter( + Tag.tag_name == value).count() + db_info = session.query(DB).outerjoin(DBTag, DB.id == DBTag.db_id).outerjoin(Tag, Tag.id == + DBTag.tag_id).filter( + Tag.tag_name == value).order_by(DB.id).offset(limit_start).limit(int(limit)) + for msg in db_info: + tag_list = [] + data_dict = model_to_dict(msg) + data_dict['create_time'] = str(data_dict['create_time']) + data_dict['update_time'] = str(data_dict['update_time']) + db_tags = session.query(Tag.tag_name).outerjoin(DBTag, Tag.id == DBTag.tag_id).filter( + DBTag.db_id == data_dict['id']).all() + for t in db_tags: + tag_list.append(t[0]) + data_dict['tag_list'] = tag_list + db_list.append(data_dict) + return self.write(dict(code=0, msg='获取成功', count=count, data=db_list)) + + ### 监听搜索 + if key and key != 'tag_name' and not value: + # TODO 超管查所有 + if self.is_superuser: + count = session.query(DB).filter(or_(DB.db_code.like('%{}%'.format(key)), + DB.db_host.like('%{}%'.format(key)), + DB.idc.like('%{}%'.format(key)), + DB.db_user.like('%{}%'.format(key)), + DB.db_pwd.like('%{}%'.format(key)), + DB.proxy_host.like('%{}%'.format(key)), + DB.db_type.like('%{}%'.format(key)), + DB.db_version.like('%{}%'.format(key)), + DB.db_mark.like('%{}%'.format(key)), + DB.state.like('%{}%'.format(key)), + DB.db_env.like('%{}%'.format(key)))).count() + + db_info = session.query(DB).filter(or_(DB.db_code.like('%{}%'.format(key)), + DB.db_host.like('%{}%'.format(key)), + DB.db_user.like('%{}%'.format(key)), + DB.db_pwd.like('%{}%'.format(key)), + DB.proxy_host.like('%{}%'.format(key)), + DB.db_type.like('%{}%'.format(key)), + DB.db_version.like('%{}%'.format(key)), + DB.db_mark.like('%{}%'.format(key)), + DB.state.like('%{}%'.format(key)), + DB.idc.like('%{}%'.format(key)), + DB.db_env.like('%{}%'.format(key)))).order_by(DB.id) + + else: + # TODO 普通用户只能看到有权限的 + db_id_list = [] + with DBContext('r') as session: + the_dbs = session.query(DBTag.db_id).filter(DBTag.tag_id.in_( + session.query(Tag.id).filter(or_(Tag.users.like('%{}%'.format(nickname)))))) + for s in the_dbs: + db_id_list.append(s[0]) + # 去重下列表,万一有重复的呢 + set_db_id_list = set(db_id_list) + count = session.query(DB).filter(DB.id.in_(set_db_id_list)).filter( + or_(DB.db_code.like('%{}%'.format(key)), + DB.db_host.like('%{}%'.format(key)), + DB.db_user.like('%{}%'.format(key)), + DB.db_pwd.like('%{}%'.format(key)), + DB.proxy_host.like('%{}%'.format(key)), + DB.db_type.like('%{}%'.format(key)), + DB.db_version.like('%{}%'.format(key)), + DB.db_mark.like('%{}%'.format(key)), + DB.state.like('%{}%'.format(key)), + DB.idc.like('%{}%'.format(key)), + DB.db_env.like('%{}%'.format(key)))).count() + + db_info = session.query(DB).filter(DB.id.in_(set_db_id_list)).filter( + or_(DB.db_code.like('%{}%'.format(key)), + DB.db_host.like('%{}%'.format(key)), + DB.db_user.like('%{}%'.format(key)), + DB.db_pwd.like('%{}%'.format(key)), + DB.proxy_host.like('%{}%'.format(key)), + DB.db_type.like('%{}%'.format(key)), + DB.db_version.like('%{}%'.format(key)), + DB.db_mark.like('%{}%'.format(key)), + DB.state.like('%{}%'.format(key)), + DB.idc.like('%{}%'.format(key)), + DB.db_env.like('%{}%'.format(key)))).order_by(DB.id).offset( + limit_start).limit(int(limit)) + + for msg in db_info: + tag_list = [] + data_dict = model_to_dict(msg) + data_dict['create_time'] = str(data_dict['create_time']) + data_dict['update_time'] = str(data_dict['update_time']) + db_tags = session.query(Tag.tag_name).outerjoin(DBTag, Tag.id == DBTag.tag_id).filter( + DBTag.db_id == data_dict['id']).all() + for t in db_tags: + tag_list.append(t[0]) + data_dict['tag_list'] = tag_list + db_list.append(data_dict) + + return self.write(dict(code=0, msg='获取成功', count=count, data=db_list)) + + ### 888查看所有的数据库 + if limit == '888': + count = session.query(DB).count() + db_info = session.query(DB).order_by(DB.id).all() + else: + # TODO 超管查所有 + if self.is_superuser: + if key and value: + count = session.query(DB).filter_by(**{key: value}).count() + db_info = session.query(DB).filter_by(**{key: value}).order_by(DB.id).offset(limit_start).limit( + int(limit)) + else: + count = session.query(DB).count() + db_info = session.query(DB).order_by(DB.id).offset(limit_start).limit(int(limit)) + else: + # TODO 普通用户只给有权限的DB,根据用户查Tagid, 根据Tagid查询出来关联的DBID，根据DBID返回主机详情 + db_id_list = [] + with DBContext('r') as session: + # 子查询查出来server_id + the_dbs = session.query(DBTag.db_id).filter(DBTag.tag_id.in_( + session.query(Tag.id).filter(or_(Tag.users.like('%{}%'.format(nickname)))))) + for d in the_dbs: + db_id_list.append(d[0]) + # 去重下列表,万一有重复的呢 + set_db_id_list = set(db_id_list) + if key and value: + # 根据Keyvalue获取 + count = session.query(DB).filter(DB.id.in_(set_db_id_list)).filter_by( + **{key: value}).count() + db_info = session.query(DB).filter(DB.id.in_(set_db_id_list)).filter_by( + **{key: value}).order_by(DB.id).offset( + limit_start).limit(int(limit)) + else: + # 获取主机详情 + + count = session.query(DB).filter(DB.id.in_(set_db_id_list)).count() + db_info = session.query(DB).filter(DB.id.in_(set_db_id_list)).offset( + limit_start).limit(int(limit)) + + for msg in db_info: + tag_list = [] + data_dict = model_to_dict(msg) + db_tags = session.query(Tag.tag_name).outerjoin(DBTag, Tag.id == DBTag.tag_id).filter( + DBTag.db_id == data_dict['id']).all() + for t in db_tags: + tag_list.append(t[0]) + + data_dict['create_time'] = str(data_dict['create_time']) + data_dict['update_time'] = str(data_dict['update_time']) + data_dict['tag_list'] = tag_list + db_list.append(data_dict) + + self.write(dict(code=0, msg='获取成功', count=count, data=db_list)) + + def post(self, *args, **kwargs): + data = json.loads(self.request.body.decode("utf-8")) + nickname = self.get_current_nickname() + db_code = data.get('db_code', None) + db_host = data.get('db_host', None) + db_port = data.get('db_port', 3306) + db_user = data.get('db_user', None) + db_pwd = data.get('db_pwd', None) + db_env = data.get('db_env', None) + proxy_host = data.get('proxy_host', None) + db_type = data.get('db_type', 'mysql') + db_version = data.get('db_version', None) + db_mark = data.get('db_mark', '写') + tag_list = data.get('tag_list', []) + db_detail = data.get('db_detail', None) + if not db_code or not db_host or not db_port or not db_user: + return self.write(dict(code=-1, msg='关键参数不能为空')) + + with DBContext('r') as session: + exist_id = session.query(DB.id).filter(DB.db_code == db_code, DB.db_host == db_host, DB.db_port == db_port, + DB.db_user == db_user, DB.db_env == db_env, + DB.proxy_host == proxy_host, DB.db_type == db_type, + db_version == db_version, + DB.db_mark == db_mark).first() + if exist_id: + return self.write(dict(code=-2, msg='不要重复记录')) + + with DBContext('w', None, True) as session: + new_db = DB(db_code=db_code, db_host=db_host, db_port=db_port, db_user=db_user, db_pwd=db_pwd, + db_env=db_env, proxy_host=proxy_host, db_type=db_type, db_version=db_version, db_mark=db_mark, + db_detail=db_detail) + session.add(new_db) + + all_tags = session.query(Tag.id).filter(Tag.tag_name.in_(tag_list)).order_by(Tag.id).all() + if all_tags: + for tag_id in all_tags: + session.add(DBTag(db_id=new_db.id, tag_id=tag_id[0])) + # 记录,记录错误也不要影响用户正常添加 + try: + with DBContext('w', None, True) as session: + + new_record = AssetOperationalAudit(username=nickname, request_object='数据库',request_host=db_host, + request_method='新增', modify_data=data) + session.add(new_record) + except Exception as err: + ins_log.read_log('error', 'operational_audit error:{err}'.format(err=err)) + + return self.write(dict(code=0, msg='添加成功')) + + def put(self, *args, **kwargs): + data = json.loads(self.request.body.decode("utf-8")) + nickname = self.get_current_nickname() + db_id = data.get('id', None) + db_code = data.get('db_code', None) + db_host = data.get('db_host', None) + db_port = data.get('db_port', 3306) + db_user = data.get('db_user', None) + db_pwd = data.get('db_pwd', None) + db_env = data.get('db_env', None) + proxy_host = data.get('proxy_host', None) + db_type = data.get('db_type', 'mysql') + db_version = data.get('db_version', None) + db_mark = data.get('db_mark', '写') + idc = data.get('idc', None) + tag_list = data.get('tag_list', []) + db_detail = data.get('db_detail', None) + if not db_id or not db_code or not db_host or not db_port or not db_user: + return self.write(dict(code=-1, msg='关键参数不能为空')) + + with DBContext('w', None, True) as session: + all_tags = session.query(Tag.id).filter(Tag.tag_name.in_(tag_list)).order_by(Tag.id).all() + session.query(DBTag).filter(DBTag.db_id == db_id).delete(synchronize_session=False) + if all_tags: + for tag_id in all_tags: + session.add(DBTag(db_id=int(db_id), tag_id=tag_id[0])) + + session.query(DB).filter(DB.id == int(db_id)).update({DB.db_code: db_code, DB.db_host: db_host, + DB.db_port: db_port, DB.db_user: db_user, + DB.db_pwd: db_pwd, DB.db_env: db_env, + DB.proxy_host: proxy_host, DB.db_type: db_type, + DB.db_version: db_version, DB.idc: idc, + DB.db_mark: db_mark, DB.db_detail: db_detail}) + # 记录操作,不成功直接Pass + try: + modify_data = data + with DBContext('w', None, True) as session: + data_info = session.query(DB).filter(DB.id == int(db_id)).all() + for data in data_info: + origion_data = model_to_dict(data) + origion_data['create_time'] = str(origion_data['create_time']) + origion_data['update_time'] = str(origion_data['update_time']) + new_record = AssetOperationalAudit(username=nickname, request_object='数据库',request_host=db_host, + request_method='更新', original_data=origion_data, + modify_data=modify_data) + session.add(new_record) + except Exception as err: + ins_log.read_log('error', 'operational_audit error:{err}'.format(err=err)) + + return self.write(dict(code=0, msg='编辑成功')) + + def delete(self, *args, **kwargs): + data = json.loads(self.request.body.decode("utf-8")) + nickname = self.get_current_nickname() + db_id = data.get('db_id', None) + id_list = data.get('id_list', None) + + + # 记录操作,不成功直接Pass + try: + with DBContext('w', None, True) as session: + if db_id: + data_info = session.query(DB).filter(DB.id == int(db_id)).all() + for data in data_info: + origion_data = model_to_dict(data) + origion_data['create_time'] = str(origion_data['create_time']) + origion_data['update_time'] = str(origion_data['update_time']) + new_record = AssetOperationalAudit(username=nickname, request_object='数据库',request_host=origion_data.get('db_host'), + request_method='删除', original_data=origion_data) + session.add(new_record) + elif id_list: + for i in id_list: + data_info = session.query(DB).filter(DB.id == i).all() + for data in data_info: + origion_data = model_to_dict(data) + origion_data['create_time'] = str(origion_data['create_time']) + origion_data['update_time'] = str(origion_data['update_time']) + new_record = AssetOperationalAudit(username=nickname, request_object='数据库',request_host=origion_data.get('db_host'), + request_method='删除', original_data=origion_data) + session.add(new_record) + except Exception as err: + ins_log.read_log('error', 'operational_audit error:{err}'.format(err=err)) + + with DBContext('w', None, True) as session: + if db_id: + session.query(DB).filter(DB.id == int(db_id)).delete(synchronize_session=False) + session.query(DBTag).filter(DBTag.db_id == int(db_id)).delete(synchronize_session=False) + elif id_list: + for i in id_list: + session.query(DB).filter(DB.id == i).delete(synchronize_session=False) + session.query(DBTag).filter(DBTag.db_id == i).delete(synchronize_session=False) + else: + return self.write(dict(code=1, msg='关键参数不能为空')) + return self.write(dict(code=0, msg='删除成功')) + + +class MultiAddDBHandler(BaseHandler): + def post(self, *args, **kwargs): + data = json.loads(self.request.body.decode("utf-8")) + nickname = self.get_current_nickname() + + # 列表包str格式，str空格切割 + if not data: + return self.write(dict(code=-1, msg='不能为空')) + + #记录下操作，即使报错也不影响程序 + try: + with DBContext('w', None, True) as session: + + new_record = AssetOperationalAudit(username=nickname, request_object='数据库', + request_method='批量添加', modify_data=data) + session.add(new_record) + except Exception as err: + ins_log.read_log('error', 'operational_audit error:{err}'.format(err=err)) + + # 判断下格式长度 + with DBContext('w', None, True) as session: + for i in data: + if i: + db_info = i.split(' ') + ins_log.read_log('info', 'MultiDB:{db_info}'.format(db_info=db_info)) + if len(db_info) != 6: + return self.write(dict(code=-2, msg='格式错误')) + + db_type = db_info[0] + db_name = db_info[1] + db_host = db_info[2] + db_port = db_info[3] + db_user = db_info[4] + db_passwd = db_info[5] + + if not type(db_port) is int and int(db_port) >= 65535: + return self.write(dict(code=-1, msg="端口格式不正确")) + + exist_id = session.query(DB.id).filter(DB.db_code == db_name).first() + exist_ip = session.query(DB.id).filter(DB.db_host == db_host).first() + if exist_id or exist_ip: + return self.write( + dict(code=-2, + msg='IP:{address}/Hostname:{hostname}已经存在,不要重复记录'.format(address=db_host, + hostname=db_name))) + + new_db = DB(db_code=db_name, db_host=db_host, db_port=db_port, db_user=db_user, db_type=db_type, + db_pwd=db_passwd, state='Handle') + session.add(new_db) + session.commit() + + return self.write(dict(code=0, msg='批量添加成功')) + + +asset_db_urls = [ + (r"/v1/cmdb/db/", DBHandler), + (r"/v1/cmdb/db/multi_add/", MultiAddDBHandler), +] +if __name__ == "__main__": + pass diff --git a/biz/handlers/asset_server_handler.py b/biz/handlers/asset_server_handler.py index ffc2a7e..dffe22d 100644 --- a/biz/handlers/asset_server_handler.py +++ b/biz/handlers/asset_server_handler.py @@ -1,657 +1,657 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# @Time : 2019/4/17 16:09 -# @Author : Fred Yangxiaofei -# @File : asset_server_handler.py -# @Role : 主机管理 - - -import json -from sqlalchemy import or_ -from libs.base_handler import BaseHandler -from models.server import Tag, ServerTag, Server, ServerDetail, AssetErrorLog, model_to_dict, AssetOperationalAudit, \ - AdminUser -from websdk.db_context import DBContext -import tornado.web -from tornado import gen -from concurrent.futures import ThreadPoolExecutor -from tornado.concurrent import run_on_executor -from libs.common import check_ip -from libs.server.sync_to_tagtree import main as sync_tag_tree -import datetime -from websdk.base_handler import LivenessProbe -from websdk.web_logs import ins_log -from opssdk.operate import MyCryptV2 -from tornado import httpclient -from settings import WEB_TERMINAL - - -class ServerHandler(BaseHandler): - def get(self, *args, **kwargs): - nickname = self.get_current_nickname() - key = self.get_argument('key', default=None, strip=True) - value = self.get_argument('value', default=None, strip=True) - page_size = self.get_argument('page', default=1, strip=True) - limit = self.get_argument('limit', default="888", strip=True) - limit_start = (int(page_size) - 1) * int(limit) - - server_list = [] - with DBContext('r') as session: - ### 通过TAG搜索 - if key == 'tag_name' and value: - count = session.query(Server).outerjoin(ServerTag, Server.id == ServerTag.server_id - ).outerjoin(Tag, Tag.id == ServerTag.tag_id).filter( - Tag.tag_name == value).count() - server_info = session.query(Server).outerjoin(ServerTag, Server.id == ServerTag.server_id - ).outerjoin(Tag, Tag.id == ServerTag.tag_id).filter( - Tag.tag_name == value).order_by(Server.id).offset(limit_start).limit(int(limit)) - - for msg in server_info: - tag_list = [] - data_dict = model_to_dict(msg) - data_dict['create_time'] = str(data_dict['create_time']) - data_dict['update_time'] = str(data_dict['update_time']) - server_tags = session.query(Tag.tag_name).outerjoin(ServerTag, Tag.id == ServerTag.tag_id).filter( - ServerTag.server_id == data_dict['id']).all() - for t in server_tags: - tag_list.append(t[0]) - data_dict['tag_list'] = tag_list - server_list.append(data_dict) - return self.write(dict(code=0, msg='获取成功', count=count, data=server_list)) - - ### 监听搜索 - if key and key != 'tag_name' and not value: - if self.is_superuser: - # TODO 超管做全局搜索 - count = session.query(Server).filter(or_(Server.hostname.like('%{}%'.format(key)), - Server.ip.like('%{}%'.format(key)), - Server.public_ip.like('%{}%'.format(key)), - Server.private_ip.like('%{}%'.format(key)), - Server.admin_user.like('%{}%'.format(key)), - Server.port.like('%{}%'.format(key)), - Server.idc.like('%{}%'.format(key)), - Server.region.like('%{}%'.format(key)), - Server.state.like('%{}%'.format(key)))).count() - server_info = session.query(Server).filter(or_(Server.hostname.like('%{}%'.format(key)), - Server.ip.like('%{}%'.format(key)), - Server.public_ip.like('%{}%'.format(key)), - Server.private_ip.like('%{}%'.format(key)), - Server.admin_user.like('%{}%'.format(key)), - Server.port.like('%{}%'.format(key)), - Server.idc.like('%{}%'.format(key)), - Server.region.like('%{}%'.format(key)), - Server.state.like('%{}%'.format(key)))).order_by( - Server.id) - else: - # TODO 普通用户做搜索 - server_id_list = [] - with DBContext('r') as session: - the_servers = session.query(ServerTag.server_id).filter(ServerTag.tag_id.in_( - session.query(Tag.id).filter(or_(Tag.users.like('%{}%'.format(nickname)))))) - for s in the_servers: - server_id_list.append(s[0]) - # 去重下列表,万一有重复的呢 - set_server_id_list = set(server_id_list) - # 获取主机详情 - count = session.query(Server).filter(Server.id.in_(set_server_id_list)).filter( - or_(Server.hostname.like('%{}%'.format(key)), - Server.ip.like('%{}%'.format(key)), - Server.public_ip.like('%{}%'.format(key)), - Server.admin_user.like('%{}%'.format(key)), - Server.port.like('%{}%'.format(key)), - Server.idc.like('%{}%'.format(key)), - Server.region.like('%{}%'.format(key)), - Server.state.like('%{}%'.format(key)))).count() - server_info = session.query(Server).filter(Server.id.in_(set_server_id_list)).filter( - or_(Server.hostname.like('%{}%'.format(key)), - Server.ip.like('%{}%'.format(key)), - Server.public_ip.like('%{}%'.format(key)), - Server.admin_user.like('%{}%'.format(key)), - Server.port.like('%{}%'.format(key)), - Server.idc.like('%{}%'.format(key)), - Server.region.like('%{}%'.format(key)), - Server.state.like('%{}%'.format(key)))).order_by( - Server.id) - - # .offset(limit_start).limit(int(limit)) #这里加上分页的话，有时候有时候在别的页面进行全局搜可能会有点小问题 - for msg in server_info: - tag_list = [] - data_dict = model_to_dict(msg) - data_dict['create_time'] = str(data_dict['create_time']) - data_dict['update_time'] = str(data_dict['update_time']) - db_tags = session.query(Tag.tag_name).outerjoin(ServerTag, Tag.id == ServerTag.tag_id).filter( - ServerTag.server_id == data_dict['id']).all() - for t in db_tags: - tag_list.append(t[0]) - - data_dict['tag_list'] = tag_list - server_list.append(data_dict) - - return self.write(dict(code=0, msg='获取成功', count=count, data=server_list)) - - if limit == "888": - ### 888查看所有 - count = session.query(Server).count() - server_info = session.query(Server).order_by(Server.id).all() - else: - ## 正常分页搜索 - # TODO 超级管理员查询所有 - if self.is_superuser: - if key and value: - count = session.query(Server).filter_by(**{key: value}).count() - server_info = session.query(Server).filter_by(**{key: value}).order_by(Server.id).offset( - limit_start).limit(int(limit)) - else: - count = session.query(Server).count() - server_info = session.query(Server).order_by(Server.id).offset(limit_start).limit(int(limit)) - else: - # TODO 普通用户只给有权限的主机,根据用户查Tagid, 根据Tagid查询出来关联的ServerID，根据ServerID返回主机详情 - server_id_list = [] - with DBContext('r') as session: - # 子查询查出来server_id - the_servers = session.query(ServerTag.server_id).filter(ServerTag.tag_id.in_( - session.query(Tag.id).filter(or_(Tag.users.like('%{}%'.format(nickname)))))) - for s in the_servers: - server_id_list.append(s[0]) - # 去重下列表,万一有重复的呢 - set_server_id_list = set(server_id_list) - if key and value: - # 根据Keyvalue获取 - count = session.query(Server).filter(Server.id.in_(set_server_id_list)).filter_by( - **{key: value}).count() - server_info = session.query(Server).filter(Server.id.in_(set_server_id_list)).filter_by( - **{key: value}).order_by(Server.id).offset( - limit_start).limit(int(limit)) - else: - # 获取主机详情 - count = session.query(Server).filter(Server.id.in_(set_server_id_list)).count() - server_info = session.query(Server).filter(Server.id.in_(set_server_id_list)).offset( - limit_start).limit(int(limit)) - - for msg in server_info: - tag_list = [] - data_dict = model_to_dict(msg) - server_tags = session.query(Tag.tag_name).outerjoin(ServerTag, Tag.id == ServerTag.tag_id).filter( - ServerTag.server_id == data_dict['id']).all() - for t in server_tags: - tag_list.append(t[0]) - data_dict['create_time'] = str(data_dict['create_time']) - data_dict['update_time'] = str(data_dict['update_time']) - data_dict['tag_list'] = tag_list - server_list.append(data_dict) - - self.write(dict(code=0, msg='获取成功', count=count, data=server_list)) - - def post(self, *args, **kwargs): - data = json.loads(self.request.body.decode("utf-8")) - nickname = self.get_current_nickname() - hostname = data.get('hostname', None) - ip = data.get('ip', None) - port = data.get('port', 22) - public_ip = data.get('public_ip', None) - idc = data.get('idc', None) - admin_user = data.get('admin_user', None) - region = data.get('region', None) - state = data.get('state', 'new') - tag_list = data.get('tag_list', []) - detail = data.get('detail', None) - - if not hostname or not ip or not port: - return self.write(dict(code=-1, msg='关键参数不能为空')) - - if not admin_user: - return self.write(dict(code=-1, msg='管理用户不能为空')) - - if not check_ip(ip): - return self.write(dict(code=-1, msg="IP格式不正确")) - - if not type(port) is int and int(port) >= 65535: - return self.write(dict(code=-1, msg="端口格式不正确")) - - with DBContext('r') as session: - exist_id = session.query(Server.id).filter(Server.hostname == hostname).first() - exist_ip = session.query(Server.id).filter(Server.ip == ip).first() - if exist_id or exist_ip: - return self.write(dict(code=-2, msg='不要重复记录')) - - with DBContext('w', None, True) as session: - new_server = Server(hostname=hostname, ip=ip, public_ip=public_ip, port=int(port), idc=idc, - admin_user=admin_user, region=region, state=state, detail=detail) - session.add(new_server) - - all_tags = session.query(Tag.id).filter(Tag.tag_name.in_(tag_list)).order_by(Tag.id).all() - # print('all_tags', all_tags) - if all_tags: - for tag_id in all_tags: - session.add(ServerTag(server_id=new_server.id, tag_id=tag_id[0])) - - # 记录,记录错误也不要影响用户正常添加 - try: - with DBContext('w', None, True) as session: - - new_record = AssetOperationalAudit(username=nickname, request_object='主机', request_host=ip, - request_method='新增', modify_data=data) - session.add(new_record) - except Exception as err: - ins_log.read_log('error', 'operational_audit error:{err}'.format(err=err)) - - return self.write(dict(code=0, msg='添加成功')) - - def put(self, *args, **kwargs): - data = json.loads(self.request.body.decode("utf-8")) - nickname = self.get_current_nickname() - server_id = data.get('id', None) - hostname = data.get('hostname', None) - ip = data.get('ip', None) - public_ip = data.get('public_ip', None) - port = data.get('port', None) - idc = data.get('idc', None) - admin_user = data.get('admin_user', None) - region = data.get('region', None) - tag_list = data.get('tag_list', []) - detail = data.get('detail', None) - - if not hostname or not ip or not port: - return self.write(dict(code=-1, msg='关键参数不能为空')) - - if not admin_user: - return self.write(dict(code=-1, msg='管理用户不能为空')) - - if not check_ip(ip): - return self.write(dict(code=-1, msg="IP格式不正确")) - - if not type(port) is int and int(port) >= 65535: - return self.write(dict(code=-1, msg="端口格式不正确")) - - with DBContext('w', None, True) as session: - exist_hostname = session.query(Server.hostname).filter(Server.id == server_id).first() - if exist_hostname[0] != hostname: - return self.write(dict(code=-2, msg='主机名不能修改')) - - session.query(ServerTag).filter(ServerTag.server_id == server_id).delete(synchronize_session=False) - all_tags = session.query(Tag.id).filter(Tag.tag_name.in_(tag_list)).order_by(Tag.id).all() - if all_tags: - for tag_id in all_tags: - session.add(ServerTag(server_id=server_id, tag_id=tag_id[0])) - - session.query(Server).filter(Server.id == server_id).update({Server.hostname: hostname, Server.ip: ip, - Server.port: int(port), - Server.public_ip: public_ip, - Server.idc: idc, - Server.admin_user: admin_user, - Server.region: region, Server.detail: detail - }) # Server.state: 'new' - # 记录操作,不成功直接Pass - try: - modify_data = data - with DBContext('w', None, True) as session: - data_info = session.query(Server).filter(Server.id == int(server_id)).all() - for data in data_info: - origion_data = model_to_dict(data) - origion_data['create_time'] = str(origion_data['create_time']) - origion_data['update_time'] = str(origion_data['update_time']) - new_record = AssetOperationalAudit(username=nickname, request_object='主机', request_host=ip, - request_method='更新', original_data=origion_data, - modify_data=modify_data) - session.add(new_record) - except Exception as err: - ins_log.read_log('error', 'operational_audit error:{err}'.format(err=err)) - return self.write(dict(code=0, msg='编辑成功')) - - def delete(self, *args, **kwargs): - data = json.loads(self.request.body.decode("utf-8")) - nickname = self.get_current_nickname() - server_id = data.get('server_id', None) - id_list = data.get('id_list', None) - - # 记录操作,不成功直接Pass - try: - with DBContext('w', None, True) as session: - if server_id: - data_info = session.query(Server).filter(Server.id == int(server_id)).all() - for data in data_info: - origion_data = model_to_dict(data) - origion_data['create_time'] = str(origion_data['create_time']) - origion_data['update_time'] = str(origion_data['update_time']) - new_record = AssetOperationalAudit(username=nickname, request_object='主机', - request_host=origion_data.get('ip'), - request_method='删除', original_data=origion_data) - session.add(new_record) - elif id_list: - for i in id_list: - data_info = session.query(Server).filter(Server.id == i).all() - for data in data_info: - origion_data = model_to_dict(data) - origion_data['create_time'] = str(origion_data['create_time']) - origion_data['update_time'] = str(origion_data['update_time']) - new_record = AssetOperationalAudit(username=nickname, request_object='主机', - request_host=origion_data.get('ip'), - request_method='删除', original_data=origion_data) - session.add(new_record) - except Exception as err: - ins_log.read_log('error', 'operational_audit error:{err}'.format(err=err)) - - with DBContext('w', None, True) as session: - if server_id: - server_info = session.query(Server).filter(Server.id == int(server_id)).first() - session.query(ServerDetail).filter(ServerDetail.ip == server_info.ip).delete(synchronize_session=False) - session.query(Server).filter(Server.id == int(server_id)).delete(synchronize_session=False) - session.query(ServerTag).filter(ServerTag.server_id == int(server_id)).delete(synchronize_session=False) - - - elif id_list: - for i in id_list: - server_info = session.query(Server).filter(Server.id == i).first() - session.query(ServerDetail).filter(ServerDetail.ip == server_info.ip).delete( - synchronize_session=False) - session.query(Server).filter(Server.id == i).delete(synchronize_session=False) - session.query(ServerTag).filter(ServerTag.server_id == i).delete(synchronize_session=False) - else: - return self.write(dict(code=1, msg='关键参数不能为空')) - return self.write(dict(code=0, msg='删除成功')) - - -class ServerDetailHandler(BaseHandler): - def get(self, *args, **kwargs): - key = self.get_argument('key', default=None, strip=True) - value = self.get_argument('value', default=None, strip=True) - page_size = self.get_argument('page', default=1, strip=True) - limit = self.get_argument('limit', default=10, strip=True) - limit_start = (int(page_size) - 1) * int(limit) - server_detail_list = [] - with DBContext('w', None, True) as session: - if key and value: - count = session.query(ServerDetail).filter_by(**{key: value}).count() - server_detail_info = session.query(ServerDetail).filter_by(**{key: value}).order_by( - ServerDetail.id) - # .offset(limit_start).limit(int(limit)) #这里加上分页的话，有时候有时候在别的页面进行全局搜可能会有点小问题 - else: - count = session.query(ServerDetail).count() - server_detail_info = session.query(ServerDetail).order_by(ServerDetail.id) - # .offset(limit_start).limit(int(limit)) #这里加上分页的话，有时候有时候在别的页面进行全局搜可能会有点小问题 - - for data in server_detail_info: - data_dict = model_to_dict(data) - data_dict['create_time'] = str(data_dict['create_time']) - data_dict['update_time'] = str(data_dict['update_time']) - server_detail_list.append(data_dict) - - if not server_detail_list: - return self.write(dict(code=-1, msg='获取失败')) - return self.write(dict(code=0, msg='获取成功', count=count, data=server_detail_list)) - - -class TreeHandler(BaseHandler): - def get(self, *args, **kwargs): - nickname = self.get_current_nickname() - _tree = [{ - "expand": True, - "title": 'root', - "children": [ - ] - }] - - with DBContext('r', None, True) as session: - if self.is_superuser: - # TODO 超管看所有Tag Tree - db_tags = session.query(Tag).order_by(Tag.id).all() - else: - # TODO 普通用户看有权限的TAG Tree - db_tags = session.query(Tag).order_by(Tag.id).filter(Tag.users.like('%{}%'.format(nickname))) - for msg in db_tags: - server_dict = {} - data_dict = model_to_dict(msg) - server_tags = session.query(ServerTag.id).outerjoin(Server, Server.id == ServerTag.server_id - ).filter(ServerTag.tag_id == msg.id).all() - server_dict['the_len'] = len(server_tags) - server_dict['title'] = data_dict['tag_name'] + ' ({})'.format(len(server_tags)) - # print(server_dict['title']) - server_dict['tag_name'] = data_dict['tag_name'] - # print(server_dict['tag_name']) - server_dict['node'] = 'root' - _tree[0]["children"].append(server_dict) - - self.write(dict(code=0, msg='获取成功', data=_tree)) - - -class AssetErrorLogHandler(BaseHandler): - def get(self, *args, **kwargs): - key = self.get_argument('key', default=None, strip=True) - value = self.get_argument('value', default=None, strip=True) - page_size = self.get_argument('page', default=1, strip=True) - limit = self.get_argument('limit', default=10, strip=True) - limit_start = (int(page_size) - 1) * int(limit) - with DBContext('r', None, True) as session: - log_list = [] - if key and value: - log_info = session.query(AssetErrorLog).filter_by(**{key: value}).order_by( - AssetErrorLog.id).offset(limit_start).limit(int(limit)) - else: - log_info = session.query(AssetErrorLog).order_by(AssetErrorLog.id).all() - for msg in log_info: - data_dict = model_to_dict(msg) - data_dict['create_time'] = str(data_dict['create_time']) - log_list.append(data_dict) - - return self.write(dict(code=0, msg='获取成功', data=log_list)) - - -class SyncServerTagTree(BaseHandler): - '''CMDB Server信息同步到TagTree，使用异步方法''' - _thread_pool = ThreadPoolExecutor(2) - - @run_on_executor(executor='_thread_pool') - def sync_task(self): - msg = sync_tag_tree() - return msg - - @gen.coroutine - def get(self, *args, **kwargs): - try: - # 超过120s 返回Timeout - msg = yield gen.with_timeout(datetime.timedelta(seconds=120), self.sync_task(), - quiet_exceptions=tornado.gen.TimeoutError) - if msg: - return self.write(dict(code=-1, msg=msg)) - except gen.TimeoutError: - return self.write(dict(code=-2, msg='TimeOut')) - return self.write(dict(code=0, msg='同步TagTree完成, 详细的机器同步信息可查看后端日志')) - - -class MultiAddServerHandler(BaseHandler): - def post(self, *args, **kwargs): - data = json.loads(self.request.body.decode("utf-8")) - nickname = self.get_current_nickname() - - # 列表包str格式，str空格切割 - if not data: - return self.write(dict(code=-1, msg='不能为空')) - - # 记录下操作，即使报错也不影响程序 - try: - with DBContext('w', None, True) as session: - - new_record = AssetOperationalAudit(username=nickname, request_object='主机', - request_method='批量添加', modify_data=data) - session.add(new_record) - except Exception as err: - ins_log.read_log('error', 'operational_audit error:{err}'.format(err=err)) - - # 判断下格式长度 - with DBContext('w', None, True) as session: - for i in data: - if i: - server_info = i.split(' ') - ins_log.read_log('info', 'MultiServer:{server_info}'.format(server_info=server_info)) - if len(server_info) != 4: - return self.write(dict(code=-2, msg='格式错误')) - hostname = server_info[0] - ip = server_info[1] - port = server_info[2] - admin_user = server_info[3] - if not check_ip(ip): - return self.write(dict(code=-1, msg="IP格式不正确")) - - if not type(port) is int and int(port) >= 65535: - return self.write(dict(code=-1, msg="端口格式不正确")) - - exist_id = session.query(Server.id).filter(Server.hostname == hostname).first() - exist_ip = session.query(Server.id).filter(Server.ip == ip).first() - if exist_id or exist_ip: - return self.write( - dict(code=-2, - msg='IP:{ip}/Hostname:{hostname}已经存在,不要重复记录'.format(ip=ip, hostname=hostname))) - - new_server = Server(hostname=hostname, ip=ip, port=int(port), admin_user=admin_user) - session.add(new_server) - - session.commit() - - return self.write(dict(code=0, msg='批量添加成功')) - - -class ConnnetServerinfoHandler(BaseHandler): - async def post(self, *args, **kwargs): - data = json.loads(self.request.body.decode("utf-8")) - nickname = self.get_current_user() #获取username 不用中文名 - # id = self.get_argument('id', default=None, strip=True) - server_id = data.get('server_id', None) - - if not server_id: - return self.write(dict(code=-1, msg='关键参数不能为空')) - - server_list = [] - admin_user_list = [] - - with DBContext('r') as session: - server_info = session.query(Server).filter(Server.id == server_id).filter() - for msg in server_info: - data_dict = model_to_dict(msg) - server_list.append(data_dict) - data_dict['create_time'] = str(data_dict['create_time']) - data_dict['update_time'] = str(data_dict['update_time']) - - if not server_list: - return self.write(dict(code=-1, msg='没有发现IP')) - - if not self.is_superuser: return self.write(dict(code=-1, msg='你不是超级超级管理员,不能使用WebTerminal')) - - admin_user = server_list[0]['admin_user'] - if not admin_user: - return self.write(dict(code=-1, msg='此机器没有配置管理用户，请先配置管理用户')) - - admin_user_info = session.query(AdminUser).filter(AdminUser.admin_user == admin_user).filter() - for admin_user_msg in admin_user_info: - data_dict = model_to_dict(admin_user_msg) - - data_dict['update_time'] = str(data_dict['update_time']) - admin_user_list.append(data_dict) - - try: - connect_server_info = dict() - connect_server_info['nickname'] = nickname - connect_server_info['ip'] = server_list[0].get('ip') - connect_server_info['public_ip'] = server_list[0].get('public_ip') - connect_server_info['private_ip'] = server_list[0].get('private_ip') - connect_server_info['port'] = server_list[0].get('port') - connect_server_info['admin_user'] = server_list[0].get('admin_user') - connect_server_info['system_user'] = admin_user_list[0].get('system_user') - connect_server_info['user_key'] = admin_user_list[0].get('user_key') - except (IndexError, KeyError) as err: - ins_log.read_log('error', '解析的时候出错:{err}'.format(err=err)) - return False - # print(connect_server_info) - # 数据加密给Jimmy - mc = MyCryptV2() - # 加密数据 - encrypt_data = mc.my_encrypt(str(connect_server_info)) - ins_log.read_log('info', '加密后的数据是：{}'.format(encrypt_data)) - # # 解密方法 - # decrypt_data = mc.my_decrypt(encrypt_data) - # # print(decrypt_data) - # print('解密后的数据是：{}'.format(json.dumps(str(decrypt_data)))) - - try: - web_terminal_initialssh = '{0}/webterminal/initialssh/'.format(WEB_TERMINAL) - print(web_terminal_initialssh) - the_body = json.dumps(encrypt_data) - # 异步HTTPClient - http_client = httpclient.AsyncHTTPClient() - # 异步http_client请求 - response = await http_client.fetch(web_terminal_initialssh, method="POST", body=the_body, - raise_error=False) - # 返回的数据都在body里面 - response_data = json.loads(response.body.decode('utf-8')) - # print('response_data', response_data) - if not response_data.get('status'): - return self.write(dict(code=1, msg=response_data.get('message'))) - - # 返回信息 - web_terminal_key = response_data['key'] - web_terminal_url = '{0}/webterminal/'.format(WEB_TERMINAL) - # res_list = [] - res_data = { - "web_terminal_url": web_terminal_url, - "web_terminal_key": web_terminal_key - } - # res_list.append(res_data) - - except Exception as e: - ins_log.read_log('error', '请求webterminal接口出错:{err}'.format(err=e)) - return self.write(dict(code=1, msg='请求webterminal接口出错:{err}'.format(err=e))) - - return self.write(dict(code=0, msg='Sucessfully', data=res_data)) - - -class WebTerminalLoginfoHandler(BaseHandler): - async def post(self, *args, **kwargs): - # data = json.loads(self.request.body.decode("utf-8")) - nickname = self.get_current_user() # 这里使用useame 不用中文 - #if not self.is_superuser: return self.write(dict(code=-1, msg='你不是超级超级管理员,不能使用WebTerminal')) - - connect_server_info = dict() - connect_server_info['nickname'] = nickname - # 数据加密给Jimmy - mc = MyCryptV2() - # 加密数据 - # encrypt_data = mc.my_encrypt(str(connect_server_info)) - encrypt_data = mc.my_encrypt(str(connect_server_info)) - print(encrypt_data) - ins_log.read_log('info', '加密后的数据是：{}'.format(encrypt_data)) - try: - web_terminal_initialssh = '{0}/webterminal/initiallogin/'.format(WEB_TERMINAL) - the_body = json.dumps(encrypt_data) - http_client = httpclient.AsyncHTTPClient() - response = await http_client.fetch(web_terminal_initialssh, method="POST", body=the_body, - raise_error=False) - response_data = json.loads(response.body.decode('utf-8')) - if not response_data.get('status'): - return self.write(dict(code=1, msg=response_data.get('message'))) - # 返回信息 - web_terminal_key = response_data['key'] - web_terminal_url = '{0}/logslist/'.format(WEB_TERMINAL) - res_data = { - "web_terminal_url": web_terminal_url, - "web_terminal_key": web_terminal_key - } - except Exception as e: - ins_log.read_log('error', '请求webterminal接口出错:{err}'.format(err=e)) - return self.write(dict(code=1, msg='请求webterminal接口出错:{err}'.format(err=e))) - - return self.write(dict(code=0, msg='Sucessfully', data=res_data)) - - -asset_server_urls = [ - (r"/v1/cmdb/server/", ServerHandler), - (r"/v1/cmdb/server_detail/", ServerDetailHandler), - (r"/v1/cmdb/tree/", TreeHandler), - (r"/v1/cmdb/error_log/", AssetErrorLogHandler), - (r"/v1/cmdb/server/sync_tagtree/", SyncServerTagTree), - (r"/v1/cmdb/server/multi_add/", MultiAddServerHandler), - (r"/v1/cmdb/server/webterminnal/", ConnnetServerinfoHandler), - (r"/v1/cmdb/server/webterminnal/loginfo/", WebTerminalLoginfoHandler), - (r"/are_you_ok/", LivenessProbe), -] - -if __name__ == "__main__": - pass +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Time : 2019/4/17 16:09 +# @Author : Fred Yangxiaofei +# @File : asset_server_handler.py +# @Role : 主机管理 + + +import json +from sqlalchemy import or_ +from libs.base_handler import BaseHandler +from models.server import Tag, ServerTag, Server, ServerDetail, AssetErrorLog, model_to_dict, AssetOperationalAudit, \ + AdminUser +from websdk.db_context import DBContext +import tornado.web +from tornado import gen +from concurrent.futures import ThreadPoolExecutor +from tornado.concurrent import run_on_executor +from libs.common import check_ip +from libs.server.sync_to_tagtree import main as sync_tag_tree +import datetime +from websdk.base_handler import LivenessProbe +from websdk.web_logs import ins_log +from opssdk.operate import MyCryptV2 +from tornado import httpclient +from settings import WEB_TERMINAL + + +class ServerHandler(BaseHandler): + def get(self, *args, **kwargs): + nickname = self.get_current_nickname() + key = self.get_argument('key', default=None, strip=True) + value = self.get_argument('value', default=None, strip=True) + page_size = self.get_argument('page', default=1, strip=True) + limit = self.get_argument('limit', default="888", strip=True) + limit_start = (int(page_size) - 1) * int(limit) + + server_list = [] + with DBContext('r') as session: + ### 通过TAG搜索 + if key == 'tag_name' and value: + count = session.query(Server).outerjoin(ServerTag, Server.id == ServerTag.server_id + ).outerjoin(Tag, Tag.id == ServerTag.tag_id).filter( + Tag.tag_name == value).count() + server_info = session.query(Server).outerjoin(ServerTag, Server.id == ServerTag.server_id + ).outerjoin(Tag, Tag.id == ServerTag.tag_id).filter( + Tag.tag_name == value).order_by(Server.id).offset(limit_start).limit(int(limit)) + + for msg in server_info: + tag_list = [] + data_dict = model_to_dict(msg) + data_dict['create_time'] = str(data_dict['create_time']) + data_dict['update_time'] = str(data_dict['update_time']) + server_tags = session.query(Tag.tag_name).outerjoin(ServerTag, Tag.id == ServerTag.tag_id).filter( + ServerTag.server_id == data_dict['id']).all() + for t in server_tags: + tag_list.append(t[0]) + data_dict['tag_list'] = tag_list + server_list.append(data_dict) + return self.write(dict(code=0, msg='获取成功', count=count, data=server_list)) + + ### 监听搜索 + if key and key != 'tag_name' and not value: + if self.is_superuser: + # TODO 超管做全局搜索 + count = session.query(Server).filter(or_(Server.hostname.like('%{}%'.format(key)), + Server.ip.like('%{}%'.format(key)), + Server.public_ip.like('%{}%'.format(key)), + Server.private_ip.like('%{}%'.format(key)), + Server.admin_user.like('%{}%'.format(key)), + Server.port.like('%{}%'.format(key)), + Server.idc.like('%{}%'.format(key)), + Server.region.like('%{}%'.format(key)), + Server.state.like('%{}%'.format(key)))).count() + server_info = session.query(Server).filter(or_(Server.hostname.like('%{}%'.format(key)), + Server.ip.like('%{}%'.format(key)), + Server.public_ip.like('%{}%'.format(key)), + Server.private_ip.like('%{}%'.format(key)), + Server.admin_user.like('%{}%'.format(key)), + Server.port.like('%{}%'.format(key)), + Server.idc.like('%{}%'.format(key)), + Server.region.like('%{}%'.format(key)), + Server.state.like('%{}%'.format(key)))).order_by( + Server.id) + else: + # TODO 普通用户做搜索 + server_id_list = [] + with DBContext('r') as session: + the_servers = session.query(ServerTag.server_id).filter(ServerTag.tag_id.in_( + session.query(Tag.id).filter(or_(Tag.users.like('%{}%'.format(nickname)))))) + for s in the_servers: + server_id_list.append(s[0]) + # 去重下列表,万一有重复的呢 + set_server_id_list = set(server_id_list) + # 获取主机详情 + count = session.query(Server).filter(Server.id.in_(set_server_id_list)).filter( + or_(Server.hostname.like('%{}%'.format(key)), + Server.ip.like('%{}%'.format(key)), + Server.public_ip.like('%{}%'.format(key)), + Server.admin_user.like('%{}%'.format(key)), + Server.port.like('%{}%'.format(key)), + Server.idc.like('%{}%'.format(key)), + Server.region.like('%{}%'.format(key)), + Server.state.like('%{}%'.format(key)))).count() + server_info = session.query(Server).filter(Server.id.in_(set_server_id_list)).filter( + or_(Server.hostname.like('%{}%'.format(key)), + Server.ip.like('%{}%'.format(key)), + Server.public_ip.like('%{}%'.format(key)), + Server.admin_user.like('%{}%'.format(key)), + Server.port.like('%{}%'.format(key)), + Server.idc.like('%{}%'.format(key)), + Server.region.like('%{}%'.format(key)), + Server.state.like('%{}%'.format(key)))).order_by( + Server.id) + + # .offset(limit_start).limit(int(limit)) #这里加上分页的话，有时候有时候在别的页面进行全局搜可能会有点小问题 + for msg in server_info: + tag_list = [] + data_dict = model_to_dict(msg) + data_dict['create_time'] = str(data_dict['create_time']) + data_dict['update_time'] = str(data_dict['update_time']) + db_tags = session.query(Tag.tag_name).outerjoin(ServerTag, Tag.id == ServerTag.tag_id).filter( + ServerTag.server_id == data_dict['id']).all() + for t in db_tags: + tag_list.append(t[0]) + + data_dict['tag_list'] = tag_list + server_list.append(data_dict) + + return self.write(dict(code=0, msg='获取成功', count=count, data=server_list)) + + if limit == "888": + ### 888查看所有 + count = session.query(Server).count() + server_info = session.query(Server).order_by(Server.id).all() + else: + ## 正常分页搜索 + # TODO 超级管理员查询所有 + if self.is_superuser: + if key and value: + count = session.query(Server).filter_by(**{key: value}).count() + server_info = session.query(Server).filter_by(**{key: value}).order_by(Server.id).offset( + limit_start).limit(int(limit)) + else: + count = session.query(Server).count() + server_info = session.query(Server).order_by(Server.id).offset(limit_start).limit(int(limit)) + else: + # TODO 普通用户只给有权限的主机,根据用户查Tagid, 根据Tagid查询出来关联的ServerID，根据ServerID返回主机详情 + server_id_list = [] + with DBContext('r') as session: + # 子查询查出来server_id + the_servers = session.query(ServerTag.server_id).filter(ServerTag.tag_id.in_( + session.query(Tag.id).filter(or_(Tag.users.like('%{}%'.format(nickname)))))) + for s in the_servers: + server_id_list.append(s[0]) + # 去重下列表,万一有重复的呢 + set_server_id_list = set(server_id_list) + if key and value: + # 根据Keyvalue获取 + count = session.query(Server).filter(Server.id.in_(set_server_id_list)).filter_by( + **{key: value}).count() + server_info = session.query(Server).filter(Server.id.in_(set_server_id_list)).filter_by( + **{key: value}).order_by(Server.id).offset( + limit_start).limit(int(limit)) + else: + # 获取主机详情 + count = session.query(Server).filter(Server.id.in_(set_server_id_list)).count() + server_info = session.query(Server).filter(Server.id.in_(set_server_id_list)).offset( + limit_start).limit(int(limit)) + + for msg in server_info: + tag_list = [] + data_dict = model_to_dict(msg) + server_tags = session.query(Tag.tag_name).outerjoin(ServerTag, Tag.id == ServerTag.tag_id).filter( + ServerTag.server_id == data_dict['id']).all() + for t in server_tags: + tag_list.append(t[0]) + data_dict['create_time'] = str(data_dict['create_time']) + data_dict['update_time'] = str(data_dict['update_time']) + data_dict['tag_list'] = tag_list + server_list.append(data_dict) + + self.write(dict(code=0, msg='获取成功', count=count, data=server_list)) + + def post(self, *args, **kwargs): + data = json.loads(self.request.body.decode("utf-8")) + nickname = self.get_current_nickname() + hostname = data.get('hostname', None) + ip = data.get('ip', None) + port = data.get('port', 22) + public_ip = data.get('public_ip', None) + idc = data.get('idc', None) + admin_user = data.get('admin_user', None) + region = data.get('region', None) + state = data.get('state', 'new') + tag_list = data.get('tag_list', []) + detail = data.get('detail', None) + + if not hostname or not ip or not port: + return self.write(dict(code=-1, msg='关键参数不能为空')) + + if not admin_user: + return self.write(dict(code=-1, msg='管理用户不能为空')) + + if not check_ip(ip): + return self.write(dict(code=-1, msg="IP格式不正确")) + + if not type(port) is int and int(port) >= 65535: + return self.write(dict(code=-1, msg="端口格式不正确")) + + with DBContext('r') as session: + exist_id = session.query(Server.id).filter(Server.hostname == hostname).first() + exist_ip = session.query(Server.id).filter(Server.ip == ip).first() + if exist_id or exist_ip: + return self.write(dict(code=-2, msg='不要重复记录')) + + with DBContext('w', None, True) as session: + new_server = Server(hostname=hostname, ip=ip, public_ip=public_ip, port=int(port), idc=idc, + admin_user=admin_user, region=region, state=state, detail=detail) + session.add(new_server) + + all_tags = session.query(Tag.id).filter(Tag.tag_name.in_(tag_list)).order_by(Tag.id).all() + # print('all_tags', all_tags) + if all_tags: + for tag_id in all_tags: + session.add(ServerTag(server_id=new_server.id, tag_id=tag_id[0])) + + # 记录,记录错误也不要影响用户正常添加 + try: + with DBContext('w', None, True) as session: + + new_record = AssetOperationalAudit(username=nickname, request_object='主机', request_host=ip, + request_method='新增', modify_data=data) + session.add(new_record) + except Exception as err: + ins_log.read_log('error', 'operational_audit error:{err}'.format(err=err)) + + return self.write(dict(code=0, msg='添加成功')) + + def put(self, *args, **kwargs): + data = json.loads(self.request.body.decode("utf-8")) + nickname = self.get_current_nickname() + server_id = data.get('id', None) + hostname = data.get('hostname', None) + ip = data.get('ip', None) + public_ip = data.get('public_ip', None) + port = data.get('port', None) + idc = data.get('idc', None) + admin_user = data.get('admin_user', None) + region = data.get('region', None) + tag_list = data.get('tag_list', []) + detail = data.get('detail', None) + + if not hostname or not ip or not port: + return self.write(dict(code=-1, msg='关键参数不能为空')) + + if not admin_user: + return self.write(dict(code=-1, msg='管理用户不能为空')) + + if not check_ip(ip): + return self.write(dict(code=-1, msg="IP格式不正确")) + + if not type(port) is int and int(port) >= 65535: + return self.write(dict(code=-1, msg="端口格式不正确")) + + with DBContext('w', None, True) as session: + exist_hostname = session.query(Server.hostname).filter(Server.id == server_id).first() + if exist_hostname[0] != hostname: + return self.write(dict(code=-2, msg='主机名不能修改')) + + session.query(ServerTag).filter(ServerTag.server_id == server_id).delete(synchronize_session=False) + all_tags = session.query(Tag.id).filter(Tag.tag_name.in_(tag_list)).order_by(Tag.id).all() + if all_tags: + for tag_id in all_tags: + session.add(ServerTag(server_id=server_id, tag_id=tag_id[0])) + + session.query(Server).filter(Server.id == server_id).update({Server.hostname: hostname, Server.ip: ip, + Server.port: int(port), + Server.public_ip: public_ip, + Server.idc: idc, + Server.admin_user: admin_user, + Server.region: region, Server.detail: detail + }) # Server.state: 'new' + # 记录操作,不成功直接Pass + try: + modify_data = data + with DBContext('w', None, True) as session: + data_info = session.query(Server).filter(Server.id == int(server_id)).all() + for data in data_info: + origion_data = model_to_dict(data) + origion_data['create_time'] = str(origion_data['create_time']) + origion_data['update_time'] = str(origion_data['update_time']) + new_record = AssetOperationalAudit(username=nickname, request_object='主机', request_host=ip, + request_method='更新', original_data=origion_data, + modify_data=modify_data) + session.add(new_record) + except Exception as err: + ins_log.read_log('error', 'operational_audit error:{err}'.format(err=err)) + return self.write(dict(code=0, msg='编辑成功')) + + def delete(self, *args, **kwargs): + data = json.loads(self.request.body.decode("utf-8")) + nickname = self.get_current_nickname() + server_id = data.get('server_id', None) + id_list = data.get('id_list', None) + + # 记录操作,不成功直接Pass + try: + with DBContext('w', None, True) as session: + if server_id: + data_info = session.query(Server).filter(Server.id == int(server_id)).all() + for data in data_info: + origion_data = model_to_dict(data) + origion_data['create_time'] = str(origion_data['create_time']) + origion_data['update_time'] = str(origion_data['update_time']) + new_record = AssetOperationalAudit(username=nickname, request_object='主机', + request_host=origion_data.get('ip'), + request_method='删除', original_data=origion_data) + session.add(new_record) + elif id_list: + for i in id_list: + data_info = session.query(Server).filter(Server.id == i).all() + for data in data_info: + origion_data = model_to_dict(data) + origion_data['create_time'] = str(origion_data['create_time']) + origion_data['update_time'] = str(origion_data['update_time']) + new_record = AssetOperationalAudit(username=nickname, request_object='主机', + request_host=origion_data.get('ip'), + request_method='删除', original_data=origion_data) + session.add(new_record) + except Exception as err: + ins_log.read_log('error', 'operational_audit error:{err}'.format(err=err)) + + with DBContext('w', None, True) as session: + if server_id: + server_info = session.query(Server).filter(Server.id == int(server_id)).first() + session.query(ServerDetail).filter(ServerDetail.ip == server_info.ip).delete(synchronize_session=False) + session.query(Server).filter(Server.id == int(server_id)).delete(synchronize_session=False) + session.query(ServerTag).filter(ServerTag.server_id == int(server_id)).delete(synchronize_session=False) + + + elif id_list: + for i in id_list: + server_info = session.query(Server).filter(Server.id == i).first() + session.query(ServerDetail).filter(ServerDetail.ip == server_info.ip).delete( + synchronize_session=False) + session.query(Server).filter(Server.id == i).delete(synchronize_session=False) + session.query(ServerTag).filter(ServerTag.server_id == i).delete(synchronize_session=False) + else: + return self.write(dict(code=1, msg='关键参数不能为空')) + return self.write(dict(code=0, msg='删除成功')) + + +class ServerDetailHandler(BaseHandler): + def get(self, *args, **kwargs): + key = self.get_argument('key', default=None, strip=True) + value = self.get_argument('value', default=None, strip=True) + page_size = self.get_argument('page', default=1, strip=True) + limit = self.get_argument('limit', default=10, strip=True) + limit_start = (int(page_size) - 1) * int(limit) + server_detail_list = [] + with DBContext('w', None, True) as session: + if key and value: + count = session.query(ServerDetail).filter_by(**{key: value}).count() + server_detail_info = session.query(ServerDetail).filter_by(**{key: value}).order_by( + ServerDetail.id) + # .offset(limit_start).limit(int(limit)) #这里加上分页的话，有时候有时候在别的页面进行全局搜可能会有点小问题 + else: + count = session.query(ServerDetail).count() + server_detail_info = session.query(ServerDetail).order_by(ServerDetail.id) + # .offset(limit_start).limit(int(limit)) #这里加上分页的话，有时候有时候在别的页面进行全局搜可能会有点小问题 + + for data in server_detail_info: + data_dict = model_to_dict(data) + data_dict['create_time'] = str(data_dict['create_time']) + data_dict['update_time'] = str(data_dict['update_time']) + server_detail_list.append(data_dict) + + if not server_detail_list: + return self.write(dict(code=-1, msg='获取失败')) + return self.write(dict(code=0, msg='获取成功', count=count, data=server_detail_list)) + + +class TreeHandler(BaseHandler): + def get(self, *args, **kwargs): + nickname = self.get_current_nickname() + _tree = [{ + "expand": True, + "title": 'root', + "children": [ + ] + }] + + with DBContext('r', None, True) as session: + if self.is_superuser: + # TODO 超管看所有Tag Tree + db_tags = session.query(Tag).order_by(Tag.id).all() + else: + # TODO 普通用户看有权限的TAG Tree + db_tags = session.query(Tag).order_by(Tag.id).filter(Tag.users.like('%{}%'.format(nickname))) + for msg in db_tags: + server_dict = {} + data_dict = model_to_dict(msg) + server_tags = session.query(ServerTag.id).outerjoin(Server, Server.id == ServerTag.server_id + ).filter(ServerTag.tag_id == msg.id).all() + server_dict['the_len'] = len(server_tags) + server_dict['title'] = data_dict['tag_name'] + ' ({})'.format(len(server_tags)) + # print(server_dict['title']) + server_dict['tag_name'] = data_dict['tag_name'] + # print(server_dict['tag_name']) + server_dict['node'] = 'root' + _tree[0]["children"].append(server_dict) + + self.write(dict(code=0, msg='获取成功', data=_tree)) + + +class AssetErrorLogHandler(BaseHandler): + def get(self, *args, **kwargs): + key = self.get_argument('key', default=None, strip=True) + value = self.get_argument('value', default=None, strip=True) + page_size = self.get_argument('page', default=1, strip=True) + limit = self.get_argument('limit', default=10, strip=True) + limit_start = (int(page_size) - 1) * int(limit) + with DBContext('r', None, True) as session: + log_list = [] + if key and value: + log_info = session.query(AssetErrorLog).filter_by(**{key: value}).order_by( + AssetErrorLog.id).offset(limit_start).limit(int(limit)) + else: + log_info = session.query(AssetErrorLog).order_by(AssetErrorLog.id).all() + for msg in log_info: + data_dict = model_to_dict(msg) + data_dict['create_time'] = str(data_dict['create_time']) + log_list.append(data_dict) + + return self.write(dict(code=0, msg='获取成功', data=log_list)) + + +class SyncServerTagTree(BaseHandler): + '''CMDB Server信息同步到TagTree，使用异步方法''' + _thread_pool = ThreadPoolExecutor(2) + + @run_on_executor(executor='_thread_pool') + def sync_task(self): + msg = sync_tag_tree() + return msg + + @gen.coroutine + def get(self, *args, **kwargs): + try: + # 超过120s 返回Timeout + msg = yield gen.with_timeout(datetime.timedelta(seconds=120), self.sync_task(), + quiet_exceptions=tornado.gen.TimeoutError) + if msg: + return self.write(dict(code=-1, msg=msg)) + except gen.TimeoutError: + return self.write(dict(code=-2, msg='TimeOut')) + return self.write(dict(code=0, msg='同步TagTree完成, 详细的机器同步信息可查看后端日志')) + + +class MultiAddServerHandler(BaseHandler): + def post(self, *args, **kwargs): + data = json.loads(self.request.body.decode("utf-8")) + nickname = self.get_current_nickname() + + # 列表包str格式，str空格切割 + if not data: + return self.write(dict(code=-1, msg='不能为空')) + + # 记录下操作，即使报错也不影响程序 + try: + with DBContext('w', None, True) as session: + + new_record = AssetOperationalAudit(username=nickname, request_object='主机', + request_method='批量添加', modify_data=data) + session.add(new_record) + except Exception as err: + ins_log.read_log('error', 'operational_audit error:{err}'.format(err=err)) + + # 判断下格式长度 + with DBContext('w', None, True) as session: + for i in data: + if i: + server_info = i.split(' ') + ins_log.read_log('info', 'MultiServer:{server_info}'.format(server_info=server_info)) + if len(server_info) != 4: + return self.write(dict(code=-2, msg='格式错误')) + hostname = server_info[0] + ip = server_info[1] + port = server_info[2] + admin_user = server_info[3] + if not check_ip(ip): + return self.write(dict(code=-1, msg="IP格式不正确")) + + if not type(port) is int and int(port) >= 65535: + return self.write(dict(code=-1, msg="端口格式不正确")) + + exist_id = session.query(Server.id).filter(Server.hostname == hostname).first() + exist_ip = session.query(Server.id).filter(Server.ip == ip).first() + if exist_id or exist_ip: + return self.write( + dict(code=-2, + msg='IP:{ip}/Hostname:{hostname}已经存在,不要重复记录'.format(ip=ip, hostname=hostname))) + + new_server = Server(hostname=hostname, ip=ip, port=int(port), admin_user=admin_user) + session.add(new_server) + + session.commit() + + return self.write(dict(code=0, msg='批量添加成功')) + + +class ConnnetServerinfoHandler(BaseHandler): + async def post(self, *args, **kwargs): + data = json.loads(self.request.body.decode("utf-8")) + nickname = self.get_current_user() #获取username 不用中文名 + # id = self.get_argument('id', default=None, strip=True) + server_id = data.get('server_id', None) + + if not server_id: + return self.write(dict(code=-1, msg='关键参数不能为空')) + + server_list = [] + admin_user_list = [] + + with DBContext('r') as session: + server_info = session.query(Server).filter(Server.id == server_id).filter() + for msg in server_info: + data_dict = model_to_dict(msg) + server_list.append(data_dict) + data_dict['create_time'] = str(data_dict['create_time']) + data_dict['update_time'] = str(data_dict['update_time']) + + if not server_list: + return self.write(dict(code=-1, msg='没有发现IP')) + + if not self.is_superuser: return self.write(dict(code=-1, msg='你不是超级超级管理员,不能使用WebTerminal')) + + admin_user = server_list[0]['admin_user'] + if not admin_user: + return self.write(dict(code=-1, msg='此机器没有配置管理用户，请先配置管理用户')) + + admin_user_info = session.query(AdminUser).filter(AdminUser.admin_user == admin_user).filter() + for admin_user_msg in admin_user_info: + data_dict = model_to_dict(admin_user_msg) + + data_dict['update_time'] = str(data_dict['update_time']) + admin_user_list.append(data_dict) + + try: + connect_server_info = dict() + connect_server_info['nickname'] = nickname + connect_server_info['ip'] = server_list[0].get('ip') + connect_server_info['public_ip'] = server_list[0].get('public_ip') + connect_server_info['private_ip'] = server_list[0].get('private_ip') + connect_server_info['port'] = server_list[0].get('port') + connect_server_info['admin_user'] = server_list[0].get('admin_user') + connect_server_info['system_user'] = admin_user_list[0].get('system_user') + connect_server_info['user_key'] = admin_user_list[0].get('user_key') + except (IndexError, KeyError) as err: + ins_log.read_log('error', '解析的时候出错:{err}'.format(err=err)) + return False + # print(connect_server_info) + # 数据加密给Jimmy + mc = MyCryptV2() + # 加密数据 + encrypt_data = mc.my_encrypt(str(connect_server_info)) + ins_log.read_log('info', '加密后的数据是：{}'.format(encrypt_data)) + # # 解密方法 + # decrypt_data = mc.my_decrypt(encrypt_data) + # # print(decrypt_data) + # print('解密后的数据是：{}'.format(json.dumps(str(decrypt_data)))) + + try: + web_terminal_initialssh = '{0}/webterminal/initialssh/'.format(WEB_TERMINAL) + print(web_terminal_initialssh) + the_body = json.dumps(encrypt_data) + # 异步HTTPClient + http_client = httpclient.AsyncHTTPClient() + # 异步http_client请求 + response = await http_client.fetch(web_terminal_initialssh, method="POST", body=the_body, + raise_error=False) + # 返回的数据都在body里面 + response_data = json.loads(response.body.decode('utf-8')) + # print('response_data', response_data) + if not response_data.get('status'): + return self.write(dict(code=1, msg=response_data.get('message'))) + + # 返回信息 + web_terminal_key = response_data['key'] + web_terminal_url = '{0}/webterminal/'.format(WEB_TERMINAL) + # res_list = [] + res_data = { + "web_terminal_url": web_terminal_url, + "web_terminal_key": web_terminal_key + } + # res_list.append(res_data) + + except Exception as e: + ins_log.read_log('error', '请求webterminal接口出错:{err}'.format(err=e)) + return self.write(dict(code=1, msg='请求webterminal接口出错:{err}'.format(err=e))) + + return self.write(dict(code=0, msg='Sucessfully', data=res_data)) + + +class WebTerminalLoginfoHandler(BaseHandler): + async def post(self, *args, **kwargs): + # data = json.loads(self.request.body.decode("utf-8")) + nickname = self.get_current_user() # 这里使用useame 不用中文 + #if not self.is_superuser: return self.write(dict(code=-1, msg='你不是超级超级管理员,不能使用WebTerminal')) + + connect_server_info = dict() + connect_server_info['nickname'] = nickname + # 数据加密给Jimmy + mc = MyCryptV2() + # 加密数据 + # encrypt_data = mc.my_encrypt(str(connect_server_info)) + encrypt_data = mc.my_encrypt(str(connect_server_info)) + print(encrypt_data) + ins_log.read_log('info', '加密后的数据是：{}'.format(encrypt_data)) + try: + web_terminal_initialssh = '{0}/webterminal/initiallogin/'.format(WEB_TERMINAL) + the_body = json.dumps(encrypt_data) + http_client = httpclient.AsyncHTTPClient() + response = await http_client.fetch(web_terminal_initialssh, method="POST", body=the_body, + raise_error=False) + response_data = json.loads(response.body.decode('utf-8')) + if not response_data.get('status'): + return self.write(dict(code=1, msg=response_data.get('message'))) + # 返回信息 + web_terminal_key = response_data['key'] + web_terminal_url = '{0}/logslist/'.format(WEB_TERMINAL) + res_data = { + "web_terminal_url": web_terminal_url, + "web_terminal_key": web_terminal_key + } + except Exception as e: + ins_log.read_log('error', '请求webterminal接口出错:{err}'.format(err=e)) + return self.write(dict(code=1, msg='请求webterminal接口出错:{err}'.format(err=e))) + + return self.write(dict(code=0, msg='Sucessfully', data=res_data)) + + +asset_server_urls = [ + (r"/v1/cmdb/server/", ServerHandler), + (r"/v1/cmdb/server_detail/", ServerDetailHandler), + (r"/v1/cmdb/tree/", TreeHandler), + (r"/v1/cmdb/error_log/", AssetErrorLogHandler), + (r"/v1/cmdb/server/sync_tagtree/", SyncServerTagTree), + (r"/v1/cmdb/server/multi_add/", MultiAddServerHandler), + (r"/v1/cmdb/server/webterminnal/", ConnnetServerinfoHandler), + (r"/v1/cmdb/server/webterminnal/loginfo/", WebTerminalLoginfoHandler), + (r"/are_you_ok/", LivenessProbe), +] + +if __name__ == "__main__": + pass diff --git a/biz/handlers/asset_tag_handler.py b/biz/handlers/asset_tag_handler.py index 65da3c5..151ccd9 100644 --- a/biz/handlers/asset_tag_handler.py +++ b/biz/handlers/asset_tag_handler.py @@ -1,501 +1,501 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# @Time : 2019/4/19 9:28 -# @Author : Fred Yangxiaofei -# @File : asset_tag_handler.py -# @Role : Tag - - -import json -from libs.base_handler import BaseHandler -from sqlalchemy import or_ -from models.server import Tag, TagRule, Server, ServerTag, model_to_dict -from models.db import DB, DBTag -from websdk.db_context import DBContext -from websdk.web_logs import ins_log -import tornado.web -from tornado import gen -from concurrent.futures import ThreadPoolExecutor -from tornado.concurrent import run_on_executor - - -class TagAuthority(BaseHandler): - def get(self, *args, **kwargs): - nickname = self.get_current_nickname() - tag_list = [] - - with DBContext('r') as session: - if self.is_superuser: - # TODO 超管看所有 - the_tags = session.query(Tag).order_by(Tag.id).all() - else: - # TODO 普通用户看有权限的TAG - the_tags = session.query(Tag).order_by(Tag.id).filter(Tag.users.like('%{}%'.format(nickname))) - - for msg in the_tags: - data_dict = model_to_dict(msg) - data_dict.pop('create_time') - if self.is_superuser: - tag_list.append(data_dict) - elif data_dict['users'] and nickname in data_dict['users'].split(','): - tag_list.append(data_dict) - return self.write(dict(code=0, msg='获取成功', data=tag_list)) - - -class TAGHandler(BaseHandler): - def get(self, *args, **kwargs): - nickname = self.get_current_nickname() - key = self.get_argument('key', default=None, strip=True) - value = self.get_argument('value', default=None, strip=True) - page_size = self.get_argument('page', default=1, strip=True) - limit = self.get_argument('limit', default="888", strip=True) - limit_start = (int(page_size) - 1) * int(limit) - tag_list = [] - - with DBContext('r') as session: - if key == 'tag_name' and value: - count = session.query(Tag).filter(Tag.tag_name.like('%{}%'.format(value))).count() - all_tags = session.query(Tag).filter(Tag.tag_name.like('%{}%'.format(value))).order_by(Tag.id).offset( - limit_start).limit(int(limit)) - elif limit == '888': - count = session.query(Tag).count() - all_tags = session.query(Tag).order_by(Tag.id).all() - elif key and key != 'tag_name' and value: - count = session.query(Tag).filter_by(**{key: value}).count() - all_tags = session.query(Tag).order_by(Tag.id).filter_by(**{key: value}).order_by(Tag.id).offset( - limit_start).limit(int(limit)) - else: - count = session.query(Tag).count() - all_tags = session.query(Tag).order_by(Tag.id).offset(limit_start).limit(int(limit)) - - for msg in all_tags: - db_list = [] - server_list = [] - data_dict = model_to_dict(msg) - data_dict['create_time'] = str(data_dict['create_time']) - if data_dict['users']: - data_dict['users'] = data_dict.get('users', '').split(',') - else: - data_dict['users'] = [] - server_tags = session.query(ServerTag.id, Server.id).outerjoin(Server, Server.id == ServerTag.server_id - ).filter( - ServerTag.tag_id == msg.id).all() - for i in server_tags: - server_list.append(i[1]) - data_dict['servers'] = server_list - data_dict['server_len'] = len(server_tags) - - db_tags = session.query(DBTag.id, DB.id, DB.db_code).outerjoin(DB, DB.id == DBTag.db_id).filter( - DBTag.tag_id == msg.id).all() - for i in db_tags: - db_list.append(i[1]) - data_dict['db_len'] = len(db_tags) - data_dict['dbs'] = db_list - tag_list.append(data_dict) - - self.write(dict(code=0, msg='获取成功', count=count, data=tag_list)) - - def post(self, *args, **kwargs): - data = json.loads(self.request.body.decode("utf-8")) - tag_name = data.get('tag_name') - users = data.get('users') - dbs = data.get('dbs') ### ID列表 - servers = data.get('servers') ### ID列表 - proxy_host = data.get('proxy_host', None) - - if not tag_name or not users: - return self.write(dict(code=-1, msg='标签名称不能为空')) - - with DBContext('r') as session: - exist_id = session.query(Tag.id).filter(Tag.tag_name == tag_name).first() - - if exist_id: - return self.write(dict(code=-2, msg='标签名称重复')) - - ### 判断是否存在 - with DBContext('w', None, True) as session: - if users: - users = ','.join(users) - new_tag = Tag(tag_name=tag_name, users=users, proxy_host=proxy_host) - session.add(new_tag) - session.commit() - if dbs: - for db in dbs: - session.add(DBTag(db_id=int(db), tag_id=new_tag.id)) - if servers: - for server in servers: - session.add(ServerTag(server_id=int(server), tag_id=new_tag.id)) - - self.write(dict(code=0, msg='添加成功')) - - def put(self, *args, **kwargs): - data = json.loads(self.request.body.decode("utf-8")) - tag_id = data.get('id') - users = data.get('users') - db_id_list = data.get('dbs') ### ID列表 - server_id_list = data.get('servers') ### ID列表 - proxy_host = data.get('proxy_host', None) - - if not users: - return self.write(dict(code=-1, msg='授权用户不能为空')) - - with DBContext('w', None, True) as session: - # if db_id_list: - # db_list, new_db_list, del_db_list = [], [], [] - # in_db_tags = session.query(DB.id).outerjoin(DBTag, DBTag.db_id == DB.id).filter( - # DBTag.tag_id == tag_id).all() - # for i in in_db_tags: - # i = i[0] - # db_list.append(i) - # if i not in db_id_list: - # del_db_list.append(i) - # session.query(DBTag).filter(DBTag.db_id == i).delete(synchronize_session=False) - # - # for i in db_id_list: - # if i not in db_list: - # session.add(DBTag(db_id=int(i), tag_id=tag_id)) - # new_db_list.append(i) - - # if server_id_list: - # server_list, new_server_list, del_server_list = [], [], [] - # in_server_tags = session.query(Server.id).outerjoin(ServerTag, ServerTag.server_id == Server.id).filter( - # ServerTag.tag_id == tag_id).all() - # - # for i in in_server_tags: - # i = i[0] - # server_list.append(i) - # if i not in server_id_list: - # del_server_list.append(i) - # - # session.query(ServerTag).filter(ServerTag.server_id == i).delete(synchronize_session=False) - # - # for i in server_id_list: - # if i not in server_list: - # session.add(ServerTag(server_id=int(i), tag_id=tag_id)) - # new_server_list.append(i) - session.query(DBTag).filter(DBTag.tag_id == int(tag_id)).delete(synchronize_session=False) - session.add_all([ - DBTag(db_id=i, tag_id=tag_id) for i in db_id_list - ]) - - session.query(ServerTag).filter(ServerTag.tag_id == int(tag_id)).delete(synchronize_session=False) - session.add_all([ - ServerTag(server_id=i, tag_id=tag_id) for i in server_id_list - ]) - if users: - users = ','.join(users) - session.query(Tag).filter(Tag.id == int(tag_id)).update({Tag.users: users, Tag.proxy_host: proxy_host}) - session.commit() - - self.write(dict(code=0, msg='修改成功')) - - def delete(self, *args, **kwargs): - data = json.loads(self.request.body.decode("utf-8")) - tag_id = data.get('tag_id') - id_list = data.get('id_list') - - with DBContext('w', None, True) as session: - if tag_id: - exist_server_count = session.query(ServerTag).filter(ServerTag.tag_id == tag_id).count() - exist_db_count = session.query(DBTag).filter(DBTag.tag_id == tag_id).count() - if exist_server_count != 0 or exist_db_count != 0: - return self.write(dict(code=1, msg='标签里面存在已关联的Server/DB，请先取消关联才能继续删除标签')) - else: - session.query(Tag).filter(Tag.id == tag_id).delete(synchronize_session=False) - - elif id_list: - for i in id_list: - exist_server_count = session.query(ServerTag).filter(ServerTag.tag_id == i).count() - exist_db_count = session.query(DBTag).filter(DBTag.tag_id == i).count() - if exist_server_count != 0 or exist_db_count != 0: - return self.write(dict(code=1, msg='标签里面存在已关联的Server/DB，请先取消关联才能继续删除标签')) - else: - session.query(Tag).filter(Tag.id == i).delete(synchronize_session=False) - else: - return self.write(dict(code=1, msg='关键参数不能为空')) - self.write(dict(code=0, msg='删除成功')) - - -class TagRuleHandler(BaseHandler): - def get(self, *args, **kwargs): - key = self.get_argument('key', default=None, strip=True) - value = self.get_argument('value', default=None, strip=True) - tag_rule_list = [] - with DBContext('r') as session: - if key and value: - tag_rule_data = session.query(TagRule).filter_by(**{key: value}).all() - else: - tag_rule_data = session.query(TagRule).all() - - for data in tag_rule_data: - data_dict = model_to_dict(data) - tag_rule_list.append(data_dict) - - return self.write(dict(code=0, msg='获取成功', data=tag_rule_list)) - - def post(self, *args, **kwargs): - data = json.loads(self.request.body.decode("utf-8")) - name = data.get('name', None) - tag_name = data.get('tag_name', None) - idc_rule = data.get('idc_rule', None) - hostname_rule = data.get('hostname_rule', None) - - if not name or not tag_name: - return self.write(dict(code=-1, msg='关键参数不能为空')) - - if not idc_rule and not hostname_rule: - return self.write(dict(code=-1, msg='一个规则条件都没创建个锤子呀')) - - with DBContext('r') as session: - exist_name = session.query(TagRule.id).filter(TagRule.name == name).first() - - if exist_name: - return self.write(dict(code=-2, msg='规则名称重复')) - - with DBContext('w', None, True) as session: - new_tag_rule = TagRule(name=name, tag_name=tag_name, idc_rule=idc_rule, hostname_rule=hostname_rule) - session.add(new_tag_rule) - session.commit() - - self.write(dict(code=0, msg='添加成功')) - - def put(self, *args, **kwargs): - data = json.loads(self.request.body.decode("utf-8")) - id = data.get('id', None) - name = data.get('name', None) - tag_name = data.get('tag_name', None) - idc_rule = data.get('idc_rule', None) - hostname_rule = data.get('hostname_rule', None) - - if not name or not tag_name: - return self.write(dict(code=-1, msg='关键参数不能为空')) - - if not idc_rule and not hostname_rule: - return self.write(dict(code=-1, msg='一个规则条件都没创建个锤子呀')) - - with DBContext('w', None, True) as session: - exist_name = session.query(TagRule.name).filter(TagRule.id == id).first() - if exist_name[0] != name: - return self.write(dict(code=-2, msg='规则名称不能修改')) - - exist_tag_name = session.query(TagRule.tag_name).filter(TagRule.id == id).first() - if exist_tag_name[0] != tag_name: - return self.write(dict(code=-2, msg='关联Tag不能修改')) - - session.query(TagRule).filter(TagRule.id == id).update( - {TagRule.idc_rule: idc_rule, TagRule.hostname_rule: hostname_rule}) # Server.state: 'new' - - return self.write(dict(code=0, msg='编辑成功')) - - def delete(self, *args, **kwargs): - data = json.loads(self.request.body.decode("utf-8")) - rule_id = data.get('id', None) - - if not rule_id: - return self.write(dict(code=-1, msg='关键参数不能为空')) - - with DBContext('w', None, True) as session: - if rule_id: - session.query(TagRule).filter(TagRule.id == int(rule_id)).delete(synchronize_session=False) - - return self.write(dict(code=0, msg='删除成功')) - - -class HandUpdateTagrule(tornado.web.RequestHandler): - '''手动触发自动规则加入tag''' - _thread_pool = ThreadPoolExecutor(2) - - @run_on_executor(executor='_thread_pool') - def hand_update_rule(self, rule_name): - with DBContext('w') as session: - tag_rule = session.query(TagRule).filter(TagRule.name == rule_name).first() - tag_rule_data = model_to_dict(tag_rule) - tag_name = tag_rule_data.get('tag_name') - idc_rule = tag_rule_data.get('idc_rule') - hostname_rule = tag_rule_data.get('hostname_rule') - - # 查到这个tag_name的tag_id - tag_id = session.query(Tag.id).filter(or_(Tag.tag_name == tag_name)).first() - tag_id = int(tag_id[0]) - - # 子查询此Tag现有的主机,肯定不能删除用户手动加的主机，不然会被打死 - tag_exist_serverid_list = session.query(ServerTag.server_id).filter(ServerTag.tag_id.in_( - session.query(Tag.id).filter(or_(Tag.tag_name == tag_name)))).all() - - tag_exist_serverid_list = [s_id[0] for s_id in tag_exist_serverid_list] - ins_log.read_log('info', '{} already exists server id list: {}'.format(tag_name, tag_exist_serverid_list)) - - if idc_rule and not hostname_rule: - # 根据idc自动匹配到的主机 - idc_matching_serverid_list = session.query(Server.id).filter(Server.idc == idc_rule).all() - idc_matching_serverid_list = [s_id[0] for s_id in idc_matching_serverid_list] - # print('根据IDC匹配到的主机列表:{}'.format(idc_matching_serverid_list)) - ins_log.read_log('info', 'IDC:{} Matching server id list: {}'.format(idc_rule, idc_matching_serverid_list)) - - # list合并 集合去重 - tocal_server_id_list = [] - tocal_server_id_list.extend(tag_exist_serverid_list) - tocal_server_id_list.extend(idc_matching_serverid_list) - tocal_server_id_list = list(set(tocal_server_id_list)) - # print('Tocal:{}'.format(tocal_server_id_list)) - ins_log.read_log('info', 'Tocal:{}'.format(tocal_server_id_list)) - - session.query(ServerTag).filter(ServerTag.tag_id == tag_id).delete() - session.add_all([ServerTag(server_id=int(i), tag_id=tag_id) for i in tocal_server_id_list]) - - - elif hostname_rule or not idc_rule: - # 根据hostname自动匹配到的主机 - hostname_matching_serverid_list = session.query(Server.id).filter( - or_(Server.hostname.like('%{}%'.format(hostname_rule)))).all() - hostname_matching_serverid_list = [s_id[0] for s_id in hostname_matching_serverid_list] - ins_log.read_log('info', - 'hostname:{} Matching server id list: {}'.format(hostname_rule, hostname_matching_serverid_list)) - # list合并 集合去重 - tocal_server_id_list = [] - tocal_server_id_list.extend(tag_exist_serverid_list) - tocal_server_id_list.extend(hostname_matching_serverid_list) - tocal_server_id_list = list(set(tocal_server_id_list)) - ins_log.read_log('info', 'Tocal:{}'.format(tocal_server_id_list)) - - # 先删除、后添加 - session.query(ServerTag).filter(ServerTag.tag_id == tag_id).delete(synchronize_session=False) - session.add_all([ServerTag(server_id=int(i), tag_id=tag_id) for i in tocal_server_id_list]) - - elif idc_rule and hostname_rule: - # 根据IDC+hostname同时匹配到的主机 - idc_matching_serverid_list = session.query(Server.id).filter(Server.idc == idc_rule).all() - idc_matching_serverid_list = [s_id[0] for s_id in idc_matching_serverid_list] - hostname_matching_serverid_list = session.query(Server.id).filter( - or_(Server.hostname.like('%{}%'.format(hostname_rule)))).all() - hostname_matching_serverid_list = [s_id[0] for s_id in hostname_matching_serverid_list] - - # list合并、集合去重 - tocal_server_id_list = [] - tocal_server_id_list.extend(idc_matching_serverid_list) - tocal_server_id_list.extend(hostname_matching_serverid_list) - tocal_server_id_list.extend(tag_exist_serverid_list) - tocal_server_id_list = list(set(tocal_server_id_list)) - ins_log.read_log('info', 'Tocal:{}'.format(tocal_server_id_list)) - # 先删除、后添加 - session.query(ServerTag).filter(ServerTag.tag_id == tag_id).delete(synchronize_session=False) - session.add_all([ServerTag(server_id=int(i), tag_id=tag_id) for i in tocal_server_id_list]) - - session.commit() - - @run_on_executor(executor='_thread_pool') - def hand_update_all_rule(self): - '''更新所有规则''' - with DBContext('w') as session: - tag_rule_list = [] - tag_rule = session.query(TagRule).all() - for r in tag_rule: - tag_rule_data = model_to_dict(r) - tag_rule_list.append(tag_rule_data) - - for rule in tag_rule_list: - tag_name = rule.get('tag_name') - idc_rule = rule.get('idc_rule') - hostname_rule = rule.get('hostname_rule') - - # 查到这个tag_name的tag_id - tag_id = session.query(Tag.id).filter(or_(Tag.tag_name == tag_name)).first() - tag_id = int(tag_id[0]) - - # 子查询此Tag现有的主机,肯定不能删除用户手动加的主机，不然会被打死 - tag_exist_serverid_list = session.query(ServerTag.server_id).filter(ServerTag.tag_id.in_( - session.query(Tag.id).filter(or_(Tag.tag_name == tag_name)))).all() - - tag_exist_serverid_list = [s_id[0] for s_id in tag_exist_serverid_list] - # print('User already exists server id list-->', tag_exist_serverid_list) - ins_log.read_log('info', - '{},already exists server id list: {}'.format(tag_name, tag_exist_serverid_list)) - if idc_rule and not hostname_rule: - # 根据idc自动匹配到的主机 - idc_matching_serverid_list = session.query(Server.id).filter(Server.idc == idc_rule).all() - idc_matching_serverid_list = [s_id[0] for s_id in idc_matching_serverid_list] - # print('根据IDC匹配到的主机列表:{}'.format(idc_matching_serverid_list)) - ins_log.read_log('info','IDC:{} Matching server id list: {}'.format(idc_rule, idc_matching_serverid_list)) - - # list合并 集合去重 - tocal_server_id_list = [] - tocal_server_id_list.extend(tag_exist_serverid_list) - tocal_server_id_list.extend(idc_matching_serverid_list) - tocal_server_id_list = list(set(tocal_server_id_list)) - # print('Tocal:{}'.format(tocal_server_id_list)) - ins_log.read_log('info', 'Tocal:{}'.format(tocal_server_id_list)) - session.query(ServerTag).filter(ServerTag.tag_id == tag_id).delete() - session.add_all([ServerTag(server_id=int(i), tag_id=tag_id) for i in tocal_server_id_list]) - - - elif hostname_rule or not idc_rule: - # 根据hostname自动匹配到的主机 - hostname_matching_serverid_list = session.query(Server.id).filter( - or_(Server.hostname.like('%{}%'.format(hostname_rule)))).all() - hostname_matching_serverid_list = [s_id[0] for s_id in hostname_matching_serverid_list] - ins_log.read_log('info', - 'hostname:{} Matching server id list: {}'.format(hostname_rule, - hostname_matching_serverid_list)) - - # list合并 集合去重 - tocal_server_id_list = [] - tocal_server_id_list.extend(tag_exist_serverid_list) - tocal_server_id_list.extend(hostname_matching_serverid_list) - tocal_server_id_list = list(set(tocal_server_id_list)) - # print('Tocal:{}'.format(tocal_server_id_list)) - ins_log.read_log('info', 'Tocal:{}'.format(tocal_server_id_list)) - - # 先删除、后添加 - session.query(ServerTag).filter(ServerTag.tag_id == tag_id).delete(synchronize_session=False) - session.add_all([ServerTag(server_id=int(i), tag_id=tag_id) for i in tocal_server_id_list]) - - elif idc_rule and hostname_rule: - # 根据IDC+hostname同时匹配到的主机 - idc_matching_serverid_list = session.query(Server.id).filter(Server.idc == idc_rule).all() - idc_matching_serverid_list = [s_id[0] for s_id in idc_matching_serverid_list] - hostname_matching_serverid_list = session.query(Server.id).filter( - or_(Server.hostname.like('%{}%'.format(hostname_rule)))).all() - hostname_matching_serverid_list = [s_id[0] for s_id in hostname_matching_serverid_list] - - # list合并、集合去重 - tocal_server_id_list = [] - tocal_server_id_list.extend(idc_matching_serverid_list) - tocal_server_id_list.extend(hostname_matching_serverid_list) - tocal_server_id_list.extend(tag_exist_serverid_list) - tocal_server_id_list = list(set(tocal_server_id_list)) - # print('Tocal：{}'.format(tocal_server_id_list)) - ins_log.read_log('info', 'Tocal:{}'.format(tocal_server_id_list)) - # 先删除、后添加 - session.query(ServerTag).filter(ServerTag.tag_id == tag_id).delete(synchronize_session=False) - session.add_all([ServerTag(server_id=int(i), tag_id=tag_id) for i in tocal_server_id_list]) - - session.commit() - - @gen.coroutine - def get(self, *args, **kwargs): - """刷新所有规则""" - yield self.hand_update_all_rule() - return self.write(dict(code=0, msg='Successful')) - - @gen.coroutine - def post(self, *args, **kwargs): - data = json.loads(self.request.body.decode("utf-8")) - rule_name = data.get('name', None) # 规则名称 - - if not rule_name: - return self.write(dict(code=-1, msg='关键参数不能为空')) - # print('手动触发规则:{name}'.format(name=rule_name)) - - yield self.hand_update_rule(rule_name) - # if err_msg: - # return self.write(dict(code=-1, msg=err_msg)) - - return self.write(dict(code=0, msg='Successful')) - - -tag_urls = [ - (r"/v1/cmdb/tag/", TAGHandler), - (r"/v1/cmdb/tag_auth/", TagAuthority), - (r"/v1/cmdb/tag_rule/", TagRuleHandler), - (r"/v1/cmdb/tag_rule/hand_update/", HandUpdateTagrule), -] +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Time : 2019/4/19 9:28 +# @Author : Fred Yangxiaofei +# @File : asset_tag_handler.py +# @Role : Tag + + +import json +from libs.base_handler import BaseHandler +from sqlalchemy import or_ +from models.server import Tag, TagRule, Server, ServerTag, model_to_dict +from models.db import DB, DBTag +from websdk.db_context import DBContext +from websdk.web_logs import ins_log +import tornado.web +from tornado import gen +from concurrent.futures import ThreadPoolExecutor +from tornado.concurrent import run_on_executor + + +class TagAuthority(BaseHandler): + def get(self, *args, **kwargs): + nickname = self.get_current_nickname() + tag_list = [] + + with DBContext('r') as session: + if self.is_superuser: + # TODO 超管看所有 + the_tags = session.query(Tag).order_by(Tag.id).all() + else: + # TODO 普通用户看有权限的TAG + the_tags = session.query(Tag).order_by(Tag.id).filter(Tag.users.like('%{}%'.format(nickname))) + + for msg in the_tags: + data_dict = model_to_dict(msg) + data_dict.pop('create_time') + if self.is_superuser: + tag_list.append(data_dict) + elif data_dict['users'] and nickname in data_dict['users'].split(','): + tag_list.append(data_dict) + return self.write(dict(code=0, msg='获取成功', data=tag_list)) + + +class TAGHandler(BaseHandler): + def get(self, *args, **kwargs): + nickname = self.get_current_nickname() + key = self.get_argument('key', default=None, strip=True) + value = self.get_argument('value', default=None, strip=True) + page_size = self.get_argument('page', default=1, strip=True) + limit = self.get_argument('limit', default="888", strip=True) + limit_start = (int(page_size) - 1) * int(limit) + tag_list = [] + + with DBContext('r') as session: + if key == 'tag_name' and value: + count = session.query(Tag).filter(Tag.tag_name.like('%{}%'.format(value))).count() + all_tags = session.query(Tag).filter(Tag.tag_name.like('%{}%'.format(value))).order_by(Tag.id).offset( + limit_start).limit(int(limit)) + elif limit == '888': + count = session.query(Tag).count() + all_tags = session.query(Tag).order_by(Tag.id).all() + elif key and key != 'tag_name' and value: + count = session.query(Tag).filter_by(**{key: value}).count() + all_tags = session.query(Tag).order_by(Tag.id).filter_by(**{key: value}).order_by(Tag.id).offset( + limit_start).limit(int(limit)) + else: + count = session.query(Tag).count() + all_tags = session.query(Tag).order_by(Tag.id).offset(limit_start).limit(int(limit)) + + for msg in all_tags: + db_list = [] + server_list = [] + data_dict = model_to_dict(msg) + data_dict['create_time'] = str(data_dict['create_time']) + if data_dict['users']: + data_dict['users'] = data_dict.get('users', '').split(',') + else: + data_dict['users'] = [] + server_tags = session.query(ServerTag.id, Server.id).outerjoin(Server, Server.id == ServerTag.server_id + ).filter( + ServerTag.tag_id == msg.id).all() + for i in server_tags: + server_list.append(i[1]) + data_dict['servers'] = server_list + data_dict['server_len'] = len(server_tags) + + db_tags = session.query(DBTag.id, DB.id, DB.db_code).outerjoin(DB, DB.id == DBTag.db_id).filter( + DBTag.tag_id == msg.id).all() + for i in db_tags: + db_list.append(i[1]) + data_dict['db_len'] = len(db_tags) + data_dict['dbs'] = db_list + tag_list.append(data_dict) + + self.write(dict(code=0, msg='获取成功', count=count, data=tag_list)) + + def post(self, *args, **kwargs): + data = json.loads(self.request.body.decode("utf-8")) + tag_name = data.get('tag_name') + users = data.get('users') + dbs = data.get('dbs') ### ID列表 + servers = data.get('servers') ### ID列表 + proxy_host = data.get('proxy_host', None) + + if not tag_name or not users: + return self.write(dict(code=-1, msg='标签名称不能为空')) + + with DBContext('r') as session: + exist_id = session.query(Tag.id).filter(Tag.tag_name == tag_name).first() + + if exist_id: + return self.write(dict(code=-2, msg='标签名称重复')) + + ### 判断是否存在 + with DBContext('w', None, True) as session: + if users: + users = ','.join(users) + new_tag = Tag(tag_name=tag_name, users=users, proxy_host=proxy_host) + session.add(new_tag) + session.commit() + if dbs: + for db in dbs: + session.add(DBTag(db_id=int(db), tag_id=new_tag.id)) + if servers: + for server in servers: + session.add(ServerTag(server_id=int(server), tag_id=new_tag.id)) + + self.write(dict(code=0, msg='添加成功')) + + def put(self, *args, **kwargs): + data = json.loads(self.request.body.decode("utf-8")) + tag_id = data.get('id') + users = data.get('users') + db_id_list = data.get('dbs') ### ID列表 + server_id_list = data.get('servers') ### ID列表 + proxy_host = data.get('proxy_host', None) + + if not users: + return self.write(dict(code=-1, msg='授权用户不能为空')) + + with DBContext('w', None, True) as session: + # if db_id_list: + # db_list, new_db_list, del_db_list = [], [], [] + # in_db_tags = session.query(DB.id).outerjoin(DBTag, DBTag.db_id == DB.id).filter( + # DBTag.tag_id == tag_id).all() + # for i in in_db_tags: + # i = i[0] + # db_list.append(i) + # if i not in db_id_list: + # del_db_list.append(i) + # session.query(DBTag).filter(DBTag.db_id == i).delete(synchronize_session=False) + # + # for i in db_id_list: + # if i not in db_list: + # session.add(DBTag(db_id=int(i), tag_id=tag_id)) + # new_db_list.append(i) + + # if server_id_list: + # server_list, new_server_list, del_server_list = [], [], [] + # in_server_tags = session.query(Server.id).outerjoin(ServerTag, ServerTag.server_id == Server.id).filter( + # ServerTag.tag_id == tag_id).all() + # + # for i in in_server_tags: + # i = i[0] + # server_list.append(i) + # if i not in server_id_list: + # del_server_list.append(i) + # + # session.query(ServerTag).filter(ServerTag.server_id == i).delete(synchronize_session=False) + # + # for i in server_id_list: + # if i not in server_list: + # session.add(ServerTag(server_id=int(i), tag_id=tag_id)) + # new_server_list.append(i) + session.query(DBTag).filter(DBTag.tag_id == int(tag_id)).delete(synchronize_session=False) + session.add_all([ + DBTag(db_id=i, tag_id=tag_id) for i in db_id_list + ]) + + session.query(ServerTag).filter(ServerTag.tag_id == int(tag_id)).delete(synchronize_session=False) + session.add_all([ + ServerTag(server_id=i, tag_id=tag_id) for i in server_id_list + ]) + if users: + users = ','.join(users) + session.query(Tag).filter(Tag.id == int(tag_id)).update({Tag.users: users, Tag.proxy_host: proxy_host}) + session.commit() + + self.write(dict(code=0, msg='修改成功')) + + def delete(self, *args, **kwargs): + data = json.loads(self.request.body.decode("utf-8")) + tag_id = data.get('tag_id') + id_list = data.get('id_list') + + with DBContext('w', None, True) as session: + if tag_id: + exist_server_count = session.query(ServerTag).filter(ServerTag.tag_id == tag_id).count() + exist_db_count = session.query(DBTag).filter(DBTag.tag_id == tag_id).count() + if exist_server_count != 0 or exist_db_count != 0: + return self.write(dict(code=1, msg='标签里面存在已关联的Server/DB，请先取消关联才能继续删除标签')) + else: + session.query(Tag).filter(Tag.id == tag_id).delete(synchronize_session=False) + + elif id_list: + for i in id_list: + exist_server_count = session.query(ServerTag).filter(ServerTag.tag_id == i).count() + exist_db_count = session.query(DBTag).filter(DBTag.tag_id == i).count() + if exist_server_count != 0 or exist_db_count != 0: + return self.write(dict(code=1, msg='标签里面存在已关联的Server/DB，请先取消关联才能继续删除标签')) + else: + session.query(Tag).filter(Tag.id == i).delete(synchronize_session=False) + else: + return self.write(dict(code=1, msg='关键参数不能为空')) + self.write(dict(code=0, msg='删除成功')) + + +class TagRuleHandler(BaseHandler): + def get(self, *args, **kwargs): + key = self.get_argument('key', default=None, strip=True) + value = self.get_argument('value', default=None, strip=True) + tag_rule_list = [] + with DBContext('r') as session: + if key and value: + tag_rule_data = session.query(TagRule).filter_by(**{key: value}).all() + else: + tag_rule_data = session.query(TagRule).all() + + for data in tag_rule_data: + data_dict = model_to_dict(data) + tag_rule_list.append(data_dict) + + return self.write(dict(code=0, msg='获取成功', data=tag_rule_list)) + + def post(self, *args, **kwargs): + data = json.loads(self.request.body.decode("utf-8")) + name = data.get('name', None) + tag_name = data.get('tag_name', None) + idc_rule = data.get('idc_rule', None) + hostname_rule = data.get('hostname_rule', None) + + if not name or not tag_name: + return self.write(dict(code=-1, msg='关键参数不能为空')) + + if not idc_rule and not hostname_rule: + return self.write(dict(code=-1, msg='一个规则条件都没创建个锤子呀')) + + with DBContext('r') as session: + exist_name = session.query(TagRule.id).filter(TagRule.name == name).first() + + if exist_name: + return self.write(dict(code=-2, msg='规则名称重复')) + + with DBContext('w', None, True) as session: + new_tag_rule = TagRule(name=name, tag_name=tag_name, idc_rule=idc_rule, hostname_rule=hostname_rule) + session.add(new_tag_rule) + session.commit() + + self.write(dict(code=0, msg='添加成功')) + + def put(self, *args, **kwargs): + data = json.loads(self.request.body.decode("utf-8")) + id = data.get('id', None) + name = data.get('name', None) + tag_name = data.get('tag_name', None) + idc_rule = data.get('idc_rule', None) + hostname_rule = data.get('hostname_rule', None) + + if not name or not tag_name: + return self.write(dict(code=-1, msg='关键参数不能为空')) + + if not idc_rule and not hostname_rule: + return self.write(dict(code=-1, msg='一个规则条件都没创建个锤子呀')) + + with DBContext('w', None, True) as session: + exist_name = session.query(TagRule.name).filter(TagRule.id == id).first() + if exist_name[0] != name: + return self.write(dict(code=-2, msg='规则名称不能修改')) + + exist_tag_name = session.query(TagRule.tag_name).filter(TagRule.id == id).first() + if exist_tag_name[0] != tag_name: + return self.write(dict(code=-2, msg='关联Tag不能修改')) + + session.query(TagRule).filter(TagRule.id == id).update( + {TagRule.idc_rule: idc_rule, TagRule.hostname_rule: hostname_rule}) # Server.state: 'new' + + return self.write(dict(code=0, msg='编辑成功')) + + def delete(self, *args, **kwargs): + data = json.loads(self.request.body.decode("utf-8")) + rule_id = data.get('id', None) + + if not rule_id: + return self.write(dict(code=-1, msg='关键参数不能为空')) + + with DBContext('w', None, True) as session: + if rule_id: + session.query(TagRule).filter(TagRule.id == int(rule_id)).delete(synchronize_session=False) + + return self.write(dict(code=0, msg='删除成功')) + + +class HandUpdateTagrule(tornado.web.RequestHandler): + '''手动触发自动规则加入tag''' + _thread_pool = ThreadPoolExecutor(2) + + @run_on_executor(executor='_thread_pool') + def hand_update_rule(self, rule_name): + with DBContext('w') as session: + tag_rule = session.query(TagRule).filter(TagRule.name == rule_name).first() + tag_rule_data = model_to_dict(tag_rule) + tag_name = tag_rule_data.get('tag_name') + idc_rule = tag_rule_data.get('idc_rule') + hostname_rule = tag_rule_data.get('hostname_rule') + + # 查到这个tag_name的tag_id + tag_id = session.query(Tag.id).filter(or_(Tag.tag_name == tag_name)).first() + tag_id = int(tag_id[0]) + + # 子查询此Tag现有的主机,肯定不能删除用户手动加的主机，不然会被打死 + tag_exist_serverid_list = session.query(ServerTag.server_id).filter(ServerTag.tag_id.in_( + session.query(Tag.id).filter(or_(Tag.tag_name == tag_name)))).all() + + tag_exist_serverid_list = [s_id[0] for s_id in tag_exist_serverid_list] + ins_log.read_log('info', '{} already exists server id list: {}'.format(tag_name, tag_exist_serverid_list)) + + if idc_rule and not hostname_rule: + # 根据idc自动匹配到的主机 + idc_matching_serverid_list = session.query(Server.id).filter(Server.idc == idc_rule).all() + idc_matching_serverid_list = [s_id[0] for s_id in idc_matching_serverid_list] + # print('根据IDC匹配到的主机列表:{}'.format(idc_matching_serverid_list)) + ins_log.read_log('info', 'IDC:{} Matching server id list: {}'.format(idc_rule, idc_matching_serverid_list)) + + # list合并 集合去重 + tocal_server_id_list = [] + tocal_server_id_list.extend(tag_exist_serverid_list) + tocal_server_id_list.extend(idc_matching_serverid_list) + tocal_server_id_list = list(set(tocal_server_id_list)) + # print('Tocal:{}'.format(tocal_server_id_list)) + ins_log.read_log('info', 'Tocal:{}'.format(tocal_server_id_list)) + + session.query(ServerTag).filter(ServerTag.tag_id == tag_id).delete() + session.add_all([ServerTag(server_id=int(i), tag_id=tag_id) for i in tocal_server_id_list]) + + + elif hostname_rule or not idc_rule: + # 根据hostname自动匹配到的主机 + hostname_matching_serverid_list = session.query(Server.id).filter( + or_(Server.hostname.like('%{}%'.format(hostname_rule)))).all() + hostname_matching_serverid_list = [s_id[0] for s_id in hostname_matching_serverid_list] + ins_log.read_log('info', + 'hostname:{} Matching server id list: {}'.format(hostname_rule, hostname_matching_serverid_list)) + # list合并 集合去重 + tocal_server_id_list = [] + tocal_server_id_list.extend(tag_exist_serverid_list) + tocal_server_id_list.extend(hostname_matching_serverid_list) + tocal_server_id_list = list(set(tocal_server_id_list)) + ins_log.read_log('info', 'Tocal:{}'.format(tocal_server_id_list)) + + # 先删除、后添加 + session.query(ServerTag).filter(ServerTag.tag_id == tag_id).delete(synchronize_session=False) + session.add_all([ServerTag(server_id=int(i), tag_id=tag_id) for i in tocal_server_id_list]) + + elif idc_rule and hostname_rule: + # 根据IDC+hostname同时匹配到的主机 + idc_matching_serverid_list = session.query(Server.id).filter(Server.idc == idc_rule).all() + idc_matching_serverid_list = [s_id[0] for s_id in idc_matching_serverid_list] + hostname_matching_serverid_list = session.query(Server.id).filter( + or_(Server.hostname.like('%{}%'.format(hostname_rule)))).all() + hostname_matching_serverid_list = [s_id[0] for s_id in hostname_matching_serverid_list] + + # list合并、集合去重 + tocal_server_id_list = [] + tocal_server_id_list.extend(idc_matching_serverid_list) + tocal_server_id_list.extend(hostname_matching_serverid_list) + tocal_server_id_list.extend(tag_exist_serverid_list) + tocal_server_id_list = list(set(tocal_server_id_list)) + ins_log.read_log('info', 'Tocal:{}'.format(tocal_server_id_list)) + # 先删除、后添加 + session.query(ServerTag).filter(ServerTag.tag_id == tag_id).delete(synchronize_session=False) + session.add_all([ServerTag(server_id=int(i), tag_id=tag_id) for i in tocal_server_id_list]) + + session.commit() + + @run_on_executor(executor='_thread_pool') + def hand_update_all_rule(self): + '''更新所有规则''' + with DBContext('w') as session: + tag_rule_list = [] + tag_rule = session.query(TagRule).all() + for r in tag_rule: + tag_rule_data = model_to_dict(r) + tag_rule_list.append(tag_rule_data) + + for rule in tag_rule_list: + tag_name = rule.get('tag_name') + idc_rule = rule.get('idc_rule') + hostname_rule = rule.get('hostname_rule') + + # 查到这个tag_name的tag_id + tag_id = session.query(Tag.id).filter(or_(Tag.tag_name == tag_name)).first() + tag_id = int(tag_id[0]) + + # 子查询此Tag现有的主机,肯定不能删除用户手动加的主机，不然会被打死 + tag_exist_serverid_list = session.query(ServerTag.server_id).filter(ServerTag.tag_id.in_( + session.query(Tag.id).filter(or_(Tag.tag_name == tag_name)))).all() + + tag_exist_serverid_list = [s_id[0] for s_id in tag_exist_serverid_list] + # print('User already exists server id list-->', tag_exist_serverid_list) + ins_log.read_log('info', + '{},already exists server id list: {}'.format(tag_name, tag_exist_serverid_list)) + if idc_rule and not hostname_rule: + # 根据idc自动匹配到的主机 + idc_matching_serverid_list = session.query(Server.id).filter(Server.idc == idc_rule).all() + idc_matching_serverid_list = [s_id[0] for s_id in idc_matching_serverid_list] + # print('根据IDC匹配到的主机列表:{}'.format(idc_matching_serverid_list)) + ins_log.read_log('info','IDC:{} Matching server id list: {}'.format(idc_rule, idc_matching_serverid_list)) + + # list合并 集合去重 + tocal_server_id_list = [] + tocal_server_id_list.extend(tag_exist_serverid_list) + tocal_server_id_list.extend(idc_matching_serverid_list) + tocal_server_id_list = list(set(tocal_server_id_list)) + # print('Tocal:{}'.format(tocal_server_id_list)) + ins_log.read_log('info', 'Tocal:{}'.format(tocal_server_id_list)) + session.query(ServerTag).filter(ServerTag.tag_id == tag_id).delete() + session.add_all([ServerTag(server_id=int(i), tag_id=tag_id) for i in tocal_server_id_list]) + + + elif hostname_rule or not idc_rule: + # 根据hostname自动匹配到的主机 + hostname_matching_serverid_list = session.query(Server.id).filter( + or_(Server.hostname.like('%{}%'.format(hostname_rule)))).all() + hostname_matching_serverid_list = [s_id[0] for s_id in hostname_matching_serverid_list] + ins_log.read_log('info', + 'hostname:{} Matching server id list: {}'.format(hostname_rule, + hostname_matching_serverid_list)) + + # list合并 集合去重 + tocal_server_id_list = [] + tocal_server_id_list.extend(tag_exist_serverid_list) + tocal_server_id_list.extend(hostname_matching_serverid_list) + tocal_server_id_list = list(set(tocal_server_id_list)) + # print('Tocal:{}'.format(tocal_server_id_list)) + ins_log.read_log('info', 'Tocal:{}'.format(tocal_server_id_list)) + + # 先删除、后添加 + session.query(ServerTag).filter(ServerTag.tag_id == tag_id).delete(synchronize_session=False) + session.add_all([ServerTag(server_id=int(i), tag_id=tag_id) for i in tocal_server_id_list]) + + elif idc_rule and hostname_rule: + # 根据IDC+hostname同时匹配到的主机 + idc_matching_serverid_list = session.query(Server.id).filter(Server.idc == idc_rule).all() + idc_matching_serverid_list = [s_id[0] for s_id in idc_matching_serverid_list] + hostname_matching_serverid_list = session.query(Server.id).filter( + or_(Server.hostname.like('%{}%'.format(hostname_rule)))).all() + hostname_matching_serverid_list = [s_id[0] for s_id in hostname_matching_serverid_list] + + # list合并、集合去重 + tocal_server_id_list = [] + tocal_server_id_list.extend(idc_matching_serverid_list) + tocal_server_id_list.extend(hostname_matching_serverid_list) + tocal_server_id_list.extend(tag_exist_serverid_list) + tocal_server_id_list = list(set(tocal_server_id_list)) + # print('Tocal：{}'.format(tocal_server_id_list)) + ins_log.read_log('info', 'Tocal:{}'.format(tocal_server_id_list)) + # 先删除、后添加 + session.query(ServerTag).filter(ServerTag.tag_id == tag_id).delete(synchronize_session=False) + session.add_all([ServerTag(server_id=int(i), tag_id=tag_id) for i in tocal_server_id_list]) + + session.commit() + + @gen.coroutine + def get(self, *args, **kwargs): + """刷新所有规则""" + yield self.hand_update_all_rule() + return self.write(dict(code=0, msg='Successful')) + + @gen.coroutine + def post(self, *args, **kwargs): + data = json.loads(self.request.body.decode("utf-8")) + rule_name = data.get('name', None) # 规则名称 + + if not rule_name: + return self.write(dict(code=-1, msg='关键参数不能为空')) + # print('手动触发规则:{name}'.format(name=rule_name)) + + yield self.hand_update_rule(rule_name) + # if err_msg: + # return self.write(dict(code=-1, msg=err_msg)) + + return self.write(dict(code=0, msg='Successful')) + + +tag_urls = [ + (r"/v1/cmdb/tag/", TAGHandler), + (r"/v1/cmdb/tag_auth/", TagAuthority), + (r"/v1/cmdb/tag_rule/", TagRuleHandler), + (r"/v1/cmdb/tag_rule/hand_update/", HandUpdateTagrule), +] diff --git a/biz/handlers/hand_update_asset_handler.py b/biz/handlers/hand_update_asset_handler.py index d9448ca..bcd3f01 100644 --- a/biz/handlers/hand_update_asset_handler.py +++ b/biz/handlers/hand_update_asset_handler.py @@ -1,62 +1,62 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# @Time : 2019/5/15 13:53 -# @Author : Fred Yangxiaofei -# @File : hand_update_asset_handler.py -# @Role : 手动更新资产 -# 防止由于每个人习惯不同，可能引起未知的异常Bug导致主程序卡死的问题,使用异步方法 - - -import json -from libs.base_handler import BaseHandler -from models.server import Server, AdminUser -from websdk.db_context import DBContext -import tornado.web -from tornado import gen -from concurrent.futures import ThreadPoolExecutor -from tornado.concurrent import run_on_executor -from libs.server.collect_asset_info import get_server_sysinfo -from libs.server.server_common import update_asset, rsync_public_key -from websdk.web_logs import ins_log -import datetime - - -class HandUpdateAssetHandler(BaseHandler): - _thread_pool = ThreadPoolExecutor(1) - - @run_on_executor(executor='_thread_pool') - def asset_update(self, id_list): - # 检查下状态，是true的话直接推送资产 - with DBContext('r', None, True) as session: - for i in id_list: - server_list = session.query(Server.ip, Server.port, AdminUser.system_user, - AdminUser.user_key, Server.state).outerjoin(AdminUser, - AdminUser.admin_user == Server.admin_user).filter( - Server.id == i).all() - # server_list = [('47.100.231.147', 22, 'root', '-----BEGIN RSA PRIVATE KEYxxxxxEND RSA PRIVATE KEY-----', 'false')] - ins_log.read_log('info', '手动触发更新资产') - rsync_sucess_list = rsync_public_key(server_list) - if rsync_sucess_list: - asset_data = get_server_sysinfo(server_list) - ins_log.read_log('info', '资产信息：{}'.format(asset_data)) - update_asset(asset_data) - - @gen.coroutine - def post(self, *args, **kwargs): - data = json.loads(self.request.body.decode("utf-8")) - id_list = data.get('id_list', None) - if not id_list: - return self.write(dict(code=1, msg='关键参数不能为空')) - - try: - # 超过120s 返回Timeout - yield gen.with_timeout(datetime.timedelta(seconds=120), [self.asset_update(id_list)], - quiet_exceptions=tornado.gen.TimeoutError) - except gen.TimeoutError: - return self.write(dict(code=-2, msg='TimeOut')) - return self.write(dict(code=0, msg='任务执行完成，提醒： 完成状态为：True, 错误状态：False， False状态下可点击查看日志进行排错')) - - -asset_hand_server_urls = [ - (r"/v1/cmdb/server/asset_update/", HandUpdateAssetHandler) -] +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Time : 2019/5/15 13:53 +# @Author : Fred Yangxiaofei +# @File : hand_update_asset_handler.py +# @Role : 手动更新资产 +# 防止由于每个人习惯不同，可能引起未知的异常Bug导致主程序卡死的问题,使用异步方法 + + +import json +from libs.base_handler import BaseHandler +from models.server import Server, AdminUser +from websdk.db_context import DBContext +import tornado.web +from tornado import gen +from concurrent.futures import ThreadPoolExecutor +from tornado.concurrent import run_on_executor +from libs.server.collect_asset_info import get_server_sysinfo +from libs.server.server_common import update_asset, rsync_public_key +from websdk.web_logs import ins_log +import datetime + + +class HandUpdateAssetHandler(BaseHandler): + _thread_pool = ThreadPoolExecutor(1) + + @run_on_executor(executor='_thread_pool') + def asset_update(self, id_list): + # 检查下状态，是true的话直接推送资产 + with DBContext('r', None, True) as session: + for i in id_list: + server_list = session.query(Server.ip, Server.port, AdminUser.system_user, + AdminUser.user_key, Server.state).outerjoin(AdminUser, + AdminUser.admin_user == Server.admin_user).filter( + Server.id == i).all() + # server_list = [('47.100.231.147', 22, 'root', '-----BEGIN RSA PRIVATE KEYxxxxxEND RSA PRIVATE KEY-----', 'false')] + ins_log.read_log('info', '手动触发更新资产') + rsync_sucess_list = rsync_public_key(server_list) + if rsync_sucess_list: + asset_data = get_server_sysinfo(server_list) + ins_log.read_log('info', '资产信息：{}'.format(asset_data)) + update_asset(asset_data) + + @gen.coroutine + def post(self, *args, **kwargs): + data = json.loads(self.request.body.decode("utf-8")) + id_list = data.get('id_list', None) + if not id_list: + return self.write(dict(code=1, msg='关键参数不能为空')) + + try: + # 超过120s 返回Timeout + yield gen.with_timeout(datetime.timedelta(seconds=120), [self.asset_update(id_list)], + quiet_exceptions=tornado.gen.TimeoutError) + except gen.TimeoutError: + return self.write(dict(code=-2, msg='TimeOut')) + return self.write(dict(code=0, msg='任务执行完成，提醒： 完成状态为：True, 错误状态：False， False状态下可点击查看日志进行排错')) + + +asset_hand_server_urls = [ + (r"/v1/cmdb/server/asset_update/", HandUpdateAssetHandler) +] diff --git a/biz/handlers/system_user_handler.py b/biz/handlers/system_user_handler.py index ab54d16..12f8b80 100644 --- a/biz/handlers/system_user_handler.py +++ b/biz/handlers/system_user_handler.py @@ -1,148 +1,148 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# @Time : 2019/5/6 13:16 -# @Author : Fred Yangxiaofei -# @File : system_user_handler.py -# @Role : 系统用户 - - -import json -from libs.base_handler import BaseHandler -from models.server import SystemUser, model_to_dict -from websdk.db_context import DBContext -from opssdk.operate import MyCryptV2 -from libs.common import exec_shell, is_number -from libs.server.push_system_user import PushSystemUser -import shortuuid - - -class SystemUserHanlder(BaseHandler): - def get(self, *args, **kwargs): - key = self.get_argument('key', default=None, strip=True) - value = self.get_argument('value', default=None, strip=True) - page_size = self.get_argument('page', default=1, strip=True) - limit = self.get_argument('limit', default=10, strip=True) - limit_start = (int(page_size) - 1) * int(limit) - system_user_list = [] - with DBContext('r', None, True) as session: - if key and value: - count = session.query(SystemUser).filter_by(**{key: value}).count() - system_user_data = session.query(SystemUser).filter_by(**{key: value}).order_by( - SystemUser.id).offset(limit_start).limit(int(limit)) - else: - count = session.query(SystemUser).count() - system_user_data = session.query(SystemUser).order_by(SystemUser.id).offset( - limit_start).limit(int(limit)) - - for data in system_user_data: - data_dict = model_to_dict(data) - data_dict['create_time'] = str(data_dict['create_time']) - data_dict['update_time'] = str(data_dict['update_time']) - if data_dict['platform_users']: - data_dict['platform_users'] = data_dict.get('platform_users', '').split(',') - system_user_list.append(data_dict) - - return self.write(dict(code=0, msg='获取成功', count=count, data=system_user_list)) - - def post(self, *args, **kwargs): - data = json.loads(self.request.body.decode("utf-8")) - name = data.get('name', None) # 名称，也是唯一 - system_user = data.get('system_user', None) # 系统用户 - platform_users = data.get('platform_users', []) # 平台用户 - priority = data.get('priority', None) # 优先级 - sudo_list = data.get('sudo_list', None) # sudo权限 - bash_shell = data.get('bash_shell', None) # sudo权限 - remarks = data.get('remarks', None) # 备注 - - if not name or not system_user or not priority or not sudo_list or not bash_shell: - return self.write(dict(code=-2, msg='关键参数不能为空')) - - if not platform_users: - return self.write(dict(code=-2, msg='请至少选择一个关联用户')) - - if not is_number(priority): - return self.write(dict(code=-2, msg='优先级必须是数字')) - - with DBContext('r', None, True) as session: - exist_id = session.query(SystemUser.id).filter(SystemUser.name == name).first() - exist_priority = session.query(SystemUser.id).filter(SystemUser.priority == int(priority)).first() - - if exist_id: - return self.write(dict(code=-2, msg='不要重复记录')) - - if exist_priority: - return self.write(dict(code=-2, msg='优先级冲突')) - - # 新建一个系统用户用来登陆主机，此用户使用密钥认证登陆主机，密钥是自动生成的，生成后保存到数据库里面 - key_name = shortuuid.uuid() - init_keygen_cmd = 'ssh-keygen -t rsa -P "" -f /tmp/{}'.format(key_name) - code, ret = exec_shell(init_keygen_cmd) - if code == 0: - # 这个系统用户的公钥和私钥是根据name+system_user生成到/tmp下的 - with open('/tmp/{}'.format(key_name), 'r') as id_rsa, open( - '/tmp/{}.pub'.format(key_name), 'r') as id_rsa_pub: - # 对密钥进行加密再写数据库 - mc = MyCryptV2() # 实例化 - _private_key = mc.my_encrypt(id_rsa.read()) - _public_key = mc.my_encrypt(id_rsa_pub.read()) - # print('加密后的id_rsa--->',_private_key) - # print('加密后的id_rsa_pub--->',_public_key) - # print('解密公钥', mc.my_decrypt(_public_key)) - # 生成密钥对写入数据库 - with DBContext('w', None, True) as session: - new_system_user = SystemUser(name=name, system_user=system_user, priority=priority, - sudo_list=sudo_list, - bash_shell=bash_shell, id_rsa=_private_key, id_rsa_pub=_public_key, - platform_users=','.join(platform_users), - remarks=remarks) - session.add(new_system_user) - return self.write(dict(code=0, msg='添加成功')) - else: - return self.write(dict(code=-4, msg=ret)) - - def put(self, *args, **kwargs): - data = json.loads(self.request.body.decode("utf-8")) - id = data.get('id', None) - name = data.get('name', None) # 名称，也是唯一 - system_user = data.get('system_user', None) # 系统用户 - platform_users = data.get('platform_users', []) # 平台用户 - priority = data.get('priority', None) # 优先级 - sudo_list = data.get('sudo_list', None) # sudo权限 - bash_shell = data.get('bash_shell', None) # sudo权限 - remarks = data.get('remarks', None) # 备注 - - if not name or not system_user or not priority or not sudo_list or not bash_shell: - return self.write(dict(code=-2, msg='关键参数不能为空')) - - if not platform_users: - return self.write(dict(code=-2, msg='请至少选择一个关联用户')) - - if not is_number(priority): - return self.write(dict(code=-2, msg='优先级必须是数字')) - - with DBContext('w', None, True) as session: - exist_sudo_list = session.query(SystemUser.sudo_list).filter(SystemUser.id == id).first() - if exist_sudo_list[0] != sudo_list: - # 存在修改sudo_list，更新新的sudo_list到主机上 - obj = PushSystemUser() - obj.update_user_sudo(system_user, sudo_list) - session.query(SystemUser).filter(SystemUser.id == id).update( - {SystemUser.platform_users: ','.join(platform_users), SystemUser.priority: priority, - SystemUser.sudo_list: sudo_list, SystemUser.remarks: remarks}) - return self.write(dict(code=0, msg='编辑成功')) - - def delete(self, *args, **kwargs): - data = json.loads(self.request.body.decode("utf-8")) - id = data.get('id') - if not id: - return self.write(dict(code=-2, msg='关键参数不能为空')) - - with DBContext('w', None, True) as session: - session.query(SystemUser).filter(SystemUser.id == id).delete(synchronize_session=False) - - self.write(dict(code=0, msg='删除成功')) - - -system_user_urls = [ - (r"/v1/cmdb/system_user/", SystemUserHanlder) -] +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Time : 2019/5/6 13:16 +# @Author : Fred Yangxiaofei +# @File : system_user_handler.py +# @Role : 系统用户 + + +import json +from libs.base_handler import BaseHandler +from models.server import SystemUser, model_to_dict +from websdk.db_context import DBContext +from opssdk.operate import MyCryptV2 +from libs.common import exec_shell, is_number +from libs.server.push_system_user import PushSystemUser +import shortuuid + + +class SystemUserHanlder(BaseHandler): + def get(self, *args, **kwargs): + key = self.get_argument('key', default=None, strip=True) + value = self.get_argument('value', default=None, strip=True) + page_size = self.get_argument('page', default=1, strip=True) + limit = self.get_argument('limit', default=10, strip=True) + limit_start = (int(page_size) - 1) * int(limit) + system_user_list = [] + with DBContext('r', None, True) as session: + if key and value: + count = session.query(SystemUser).filter_by(**{key: value}).count() + system_user_data = session.query(SystemUser).filter_by(**{key: value}).order_by( + SystemUser.id).offset(limit_start).limit(int(limit)) + else: + count = session.query(SystemUser).count() + system_user_data = session.query(SystemUser).order_by(SystemUser.id).offset( + limit_start).limit(int(limit)) + + for data in system_user_data: + data_dict = model_to_dict(data) + data_dict['create_time'] = str(data_dict['create_time']) + data_dict['update_time'] = str(data_dict['update_time']) + if data_dict['platform_users']: + data_dict['platform_users'] = data_dict.get('platform_users', '').split(',') + system_user_list.append(data_dict) + + return self.write(dict(code=0, msg='获取成功', count=count, data=system_user_list)) + + def post(self, *args, **kwargs): + data = json.loads(self.request.body.decode("utf-8")) + name = data.get('name', None) # 名称，也是唯一 + system_user = data.get('system_user', None) # 系统用户 + platform_users = data.get('platform_users', []) # 平台用户 + priority = data.get('priority', None) # 优先级 + sudo_list = data.get('sudo_list', None) # sudo权限 + bash_shell = data.get('bash_shell', None) # sudo权限 + remarks = data.get('remarks', None) # 备注 + + if not name or not system_user or not priority or not sudo_list or not bash_shell: + return self.write(dict(code=-2, msg='关键参数不能为空')) + + if not platform_users: + return self.write(dict(code=-2, msg='请至少选择一个关联用户')) + + if not is_number(priority): + return self.write(dict(code=-2, msg='优先级必须是数字')) + + with DBContext('r', None, True) as session: + exist_id = session.query(SystemUser.id).filter(SystemUser.name == name).first() + exist_priority = session.query(SystemUser.id).filter(SystemUser.priority == int(priority)).first() + + if exist_id: + return self.write(dict(code=-2, msg='不要重复记录')) + + if exist_priority: + return self.write(dict(code=-2, msg='优先级冲突')) + + # 新建一个系统用户用来登陆主机，此用户使用密钥认证登陆主机，密钥是自动生成的，生成后保存到数据库里面 + key_name = shortuuid.uuid() + init_keygen_cmd = 'ssh-keygen -t rsa -P "" -f /tmp/{}'.format(key_name) + code, ret = exec_shell(init_keygen_cmd) + if code == 0: + # 这个系统用户的公钥和私钥是根据name+system_user生成到/tmp下的 + with open('/tmp/{}'.format(key_name), 'r') as id_rsa, open( + '/tmp/{}.pub'.format(key_name), 'r') as id_rsa_pub: + # 对密钥进行加密再写数据库 + mc = MyCryptV2() # 实例化 + _private_key = mc.my_encrypt(id_rsa.read()) + _public_key = mc.my_encrypt(id_rsa_pub.read()) + # print('加密后的id_rsa--->',_private_key) + # print('加密后的id_rsa_pub--->',_public_key) + # print('解密公钥', mc.my_decrypt(_public_key)) + # 生成密钥对写入数据库 + with DBContext('w', None, True) as session: + new_system_user = SystemUser(name=name, system_user=system_user, priority=priority, + sudo_list=sudo_list, + bash_shell=bash_shell, id_rsa=_private_key, id_rsa_pub=_public_key, + platform_users=','.join(platform_users), + remarks=remarks) + session.add(new_system_user) + return self.write(dict(code=0, msg='添加成功')) + else: + return self.write(dict(code=-4, msg=ret)) + + def put(self, *args, **kwargs): + data = json.loads(self.request.body.decode("utf-8")) + id = data.get('id', None) + name = data.get('name', None) # 名称，也是唯一 + system_user = data.get('system_user', None) # 系统用户 + platform_users = data.get('platform_users', []) # 平台用户 + priority = data.get('priority', None) # 优先级 + sudo_list = data.get('sudo_list', None) # sudo权限 + bash_shell = data.get('bash_shell', None) # sudo权限 + remarks = data.get('remarks', None) # 备注 + + if not name or not system_user or not priority or not sudo_list or not bash_shell: + return self.write(dict(code=-2, msg='关键参数不能为空')) + + if not platform_users: + return self.write(dict(code=-2, msg='请至少选择一个关联用户')) + + if not is_number(priority): + return self.write(dict(code=-2, msg='优先级必须是数字')) + + with DBContext('w', None, True) as session: + exist_sudo_list = session.query(SystemUser.sudo_list).filter(SystemUser.id == id).first() + if exist_sudo_list[0] != sudo_list: + # 存在修改sudo_list，更新新的sudo_list到主机上 + obj = PushSystemUser() + obj.update_user_sudo(system_user, sudo_list) + session.query(SystemUser).filter(SystemUser.id == id).update( + {SystemUser.platform_users: ','.join(platform_users), SystemUser.priority: priority, + SystemUser.sudo_list: sudo_list, SystemUser.remarks: remarks}) + return self.write(dict(code=0, msg='编辑成功')) + + def delete(self, *args, **kwargs): + data = json.loads(self.request.body.decode("utf-8")) + id = data.get('id') + if not id: + return self.write(dict(code=-2, msg='关键参数不能为空')) + + with DBContext('w', None, True) as session: + session.query(SystemUser).filter(SystemUser.id == id).delete(synchronize_session=False) + + self.write(dict(code=0, msg='删除成功')) + + +system_user_urls = [ + (r"/v1/cmdb/system_user/", SystemUserHanlder) +] diff --git a/doc/README.md b/doc/README.md deleted file mode 100644 index 565cf4c..0000000 --- a/doc/README.md +++ /dev/null @@ -1,12 +0,0 @@ -### 更新 - -- beta0.3.1 支持了华为云资产获取 - -``` -前端更换成最新的前端、更改表结构即可 -ALTER TABLE `asset_configs` ADD `project_id` VARCHAR(120) NOT NULL ; -ALTER TABLE `asset_configs` ADD `huawei_cloud` VARCHAR(120) NOT NULL ; -ALTER TABLE `asset_configs` ADD `huawei_instance_id` VARCHAR(120) NOT NULL ; - -#若是不想更改表结构、或者是第一次部署的同学，默认docker exec -ti cmdb_codo_cmdb_1 /usr/local/bin/python3 /var/www/codo-cmdb/db_sync.py 初始化表结构即可 -``` diff --git a/doc/nginx_ops.conf b/doc/nginx_ops.conf deleted file mode 100644 index 0b5e71a..0000000 --- a/doc/nginx_ops.conf +++ /dev/null @@ -1,22 +0,0 @@ -upstream codo-cmdb{ - server 127.0.0.1:9000; - server 127.0.0.1:9001; - server 127.0.0.1:9002; -} - - -server -{ - listen 80; - server_name cmdb2.opendevops.cn; - access_log /var/log/nginx/codo-cmdb_access.log; - error_log /var/log/nginx/codo-cmdb_error.log; - location / { - proxy_set_header Host $http_host; - proxy_redirect off; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Scheme $scheme; - proxy_set_header Cookie $http_cookie; - proxy_pass http://codo-cmdb; - } -} \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index 6452612..0000000 --- a/docker-compose.yml +++ /dev/null @@ -1,9 +0,0 @@ -codo_cmdb: - restart: unless-stopped - image: codo_cmdb:latest - volumes: - - /var/log/supervisor/:/var/log/supervisor/ - - /sys/fs/cgroup:/sys/fs/cgroup - ports: - - "8050:80" - hostname: codo-cmdb \ No newline at end of file diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml new file mode 100644 index 0000000..d471a5f --- /dev/null +++ b/docker/docker-compose.yml @@ -0,0 +1,17 @@ +version: "3.8" +networks: + codo: +services: + codo_cmdb: + env_file: + - ./env.sh + restart: unless-stopped + image: registry.cn-hangzhou.aliyuncs.com/sourcegarden/opendevops-cmdb:alpine-0.3.0 + volumes: + - /var/log/supervisor/:/var/log/supervisor/ + - /sys/fs/cgroup:/sys/fs/cgroup + ports: + - "8050:80" + hostname: codo-cmdb + networks: + - codo \ No newline at end of file diff --git a/docker/nginx_default.conf b/docker/nginx_default.conf new file mode 100644 index 0000000..e5709d3 --- /dev/null +++ b/docker/nginx_default.conf @@ -0,0 +1,64 @@ +user nginx; +worker_processes auto; +error_log /var/log/nginx/error.log; +pid /run/nginx.pid; + +# Load dynamic modules. See /usr/share/doc/nginx/README.dynamic. +include /usr/share/nginx/modules/*.conf; + +events { + worker_connections 1024; +} + +http { + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + tcp_nopush on; + tcp_nodelay on; + keepalive_timeout 65; + types_hash_max_size 2048; + + include /etc/nginx/mime.types; + default_type application/octet-stream; + + # Load modular configuration files from the /etc/nginx/conf.d directory. + # See http://nginx.org/en/docs/ngx_core_module.html#include + # for more information. + include /etc/nginx/conf.d/*.conf; + +# Settings for a TLS enabled server. +# +# server { +# listen 443 ssl http2 default_server; +# listen [::]:443 ssl http2 default_server; +# server_name _; +# root /usr/share/nginx/html; +# +# ssl_certificate "/etc/pki/nginx/server.crt"; +# ssl_certificate_key "/etc/pki/nginx/private/server.key"; +# ssl_session_cache shared:SSL:1m; +# ssl_session_timeout 10m; +# ssl_ciphers HIGH:!aNULL:!MD5; +# ssl_prefer_server_ciphers on; +# +# # Load configuration files for the default server block. +# include /etc/nginx/default.d/*.conf; +# +# location / { +# } +# +# error_page 404 /404.html; +# location = /40x.html { +# } +# +# error_page 500 502 503 504 /50x.html; +# location = /50x.html { +# } +# } + +} \ No newline at end of file diff --git a/docker/nginx_ops.conf b/docker/nginx_ops.conf new file mode 100644 index 0000000..8b6759e --- /dev/null +++ b/docker/nginx_ops.conf @@ -0,0 +1,22 @@ +upstream codo-cmdb-nginx{ + server 127.0.0.1:9000; + server 127.0.0.1:9001; + server 127.0.0.1:9002; +} + + +server +{ + listen 80; + server_name cmdb.opendevops.cn; + access_log /var/log/nginx/codo-cmdb-access.log; + error_log /var/log/nginx/codo-cmdb-error.log; + location / { + proxy_set_header Host $http_host; + proxy_redirect off; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Scheme $scheme; + proxy_set_header Cookie $http_cookie; + proxy_pass http://codo-cmdb-nginx; + } +} \ No newline at end of file diff --git a/docker/prestart.sh b/docker/prestart.sh new file mode 100644 index 0000000..79ed429 --- /dev/null +++ b/docker/prestart.sh @@ -0,0 +1,40 @@ +#!/bin/sh +cd /var/www/codo-cmdb/ + echo -e "\033[32m [INFO]: 开始修改各项目的配置文件 \033[0m" + #资产管理-cmdb + CMDB_DB_DBNAME='codo_cmdb' + sed -i "s#cookie_secret = .*#cookie_secret = '${cookie_secret}'#g" codo-cmdb-settings.py + sed -i "s#DEFAULT_DB_DBHOST = .*#DEFAULT_DB_DBHOST = os.getenv('DEFAULT_DB_DBHOST', '${DEFAULT_DB_DBHOST}')#g" codo-cmdb-settings.py + sed -i "s#DEFAULT_DB_DBPORT = .*#DEFAULT_DB_DBPORT = os.getenv('DEFAULT_DB_DBPORT', '${DEFAULT_DB_DBPORT}')#g" codo-cmdb-settings.py + sed -i "s#DEFAULT_DB_DBUSER = .*#DEFAULT_DB_DBUSER = os.getenv('DEFAULT_DB_DBUSER', '${DEFAULT_DB_DBUSER}')#g" codo-cmdb-settings.py + sed -i "s#DEFAULT_DB_DBPWD = .*#DEFAULT_DB_DBPWD = os.getenv('DEFAULT_DB_DBPWD', '${DEFAULT_DB_DBPWD}')#g" codo-cmdb-settings.py + sed -i "s#DEFAULT_DB_DBNAME = .*#DEFAULT_DB_DBNAME = os.getenv('DEFAULT_DB_DBNAME', '${CMDB_DB_DBNAME}')#g" codo-cmdb-settings.py + sed -i "s#READONLY_DB_DBHOST = .*#READONLY_DB_DBHOST = os.getenv('READONLY_DB_DBHOST', '${READONLY_DB_DBHOST}')#g" codo-cmdb-settings.py + sed -i "s#READONLY_DB_DBPORT = .*#READONLY_DB_DBPORT = os.getenv('READONLY_DB_DBPORT', '${READONLY_DB_DBPORT}')#g" codo-cmdb-settings.py + sed -i "s#READONLY_DB_DBUSER = .*#READONLY_DB_DBUSER = os.getenv('READONLY_DB_DBUSER', '${READONLY_DB_DBUSER}')#g" codo-cmdb-settings.py + sed -i "s#READONLY_DB_DBPWD = .*#READONLY_DB_DBPWD = os.getenv('READONLY_DB_DBPWD', '${READONLY_DB_DBPWD}')#g" codo-cmdb-settings.py + sed -i "s#READONLY_DB_DBNAME = .*#READONLY_DB_DBNAME = os.getenv('READONLY_DB_DBNAME', '${CMDB_DB_DBNAME}')#g" codo-cmdb-settings.py + sed -i "s#DEFAULT_REDIS_HOST = .*#DEFAULT_REDIS_HOST = os.getenv('DEFAULT_REDIS_HOST', '${DEFAULT_REDIS_HOST}')#g" codo-cmdb-settings.py + sed -i "s#DEFAULT_REDIS_PORT = .*#DEFAULT_REDIS_PORT = os.getenv('DEFAULT_REDIS_PORT', '${DEFAULT_REDIS_PORT}')#g" codo-cmdb-settings.py + sed -i "s#DEFAULT_REDIS_PASSWORD = .*#DEFAULT_REDIS_PASSWORD = os.getenv('DEFAULT_REDIS_PASSWORD', '${DEFAULT_REDIS_PASSWORD}')#g" codo-cmdb-settings.py + # 同步TAG树 + sed -i "s#CODO_TASK_DB_HOST = .*#CODO_TASK_DB_HOST = os.getenv('CODO_TASK_DB_HOST', '${DEFAULT_DB_DBHOST}')#g" codo-cmdb-settings.py + sed -i "s#CODO_TASK_DB_PORT = .*#CODO_TASK_DB_PORT = os.getenv('CODO_TASK_DB_PORT', '${DEFAULT_DB_DBPORT}')#g" codo-cmdb-settings.py + sed -i "s#CODO_TASK_DB_USER = .*#CODO_TASK_DB_USER = os.getenv('CODO_TASK_DB_USER', '${DEFAULT_DB_DBUSER}')#g" codo-cmdb-settings.py + sed -i "s#CODO_TASK_DB_PWD = .*#CODO_TASK_DB_PWD = os.getenv('CODO_TASK_DB_PWD', '${DEFAULT_DB_DBPWD}')#g" codo-cmdb-settings.py + sed -i "s#CODO_TASK_DB_DBNAME = .*#CODO_TASK_DB_DBNAME = os.getenv('CODO_TASK_DB_DBNAME', '${TASK_DB_DBNAME}')#g" codo-cmdb-settings.py + + +try_num=0 + +while [[ $try_num -le 100 ]]; +do + if $(curl -s ${DEFAULT_DB_DBHOST}:${DEFAULT_DB_DBPORT} > /dev/null);then + python3 db_sync.py + exit 0 + else + echo 'wait mysql start to do db_sync.db' + fi + let try_num+=1 + sleep 6 +done diff --git a/doc/supervisor_ops.conf b/docker/supervisor_ops.conf similarity index 72% rename from doc/supervisor_ops.conf rename to docker/supervisor_ops.conf index 96f8723..ea97e43 100644 --- a/doc/supervisor_ops.conf +++ b/docker/supervisor_ops.conf @@ -2,7 +2,19 @@ nodaemon=true [group:codo_cmdb,] -programs=cmdb,cmdb_cron +programs=cmdb,cmdb_cron,cmdb-prestart + +[program:cmdb-prestart] +command=sh docker/prestart.sh +directory=/var/www/codo-cmdb +user=root +autostart = true +autorestart=false +redirect_stderr=true +stdout_logfile=/var/log/supervisor/codo-cmdb-prestart.log +loglevel=info +logfile_maxbytes=10MB +logfile_backups=1 [program:cmdb] command=python3 startup.py --service=cmdb --port=90%(process_num)02d diff --git a/libs/aliyun/__init__.py b/libs/aliyun/__init__.py index 5a3b627..a29d424 100644 --- a/libs/aliyun/__init__.py +++ b/libs/aliyun/__init__.py @@ -1,6 +1,6 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# @Time : 2019/5/13 14:01 -# @Author : Fred Yangxiaofei -# @File : __init__.py.py +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Time : 2019/5/13 14:01 +# @Author : Fred Yangxiaofei +# @File : __init__.py.py # @Role : 说明脚本功能 \ No newline at end of file diff --git a/libs/aliyun/ecs.py b/libs/aliyun/ecs.py index 603d4c3..bc14241 100644 --- a/libs/aliyun/ecs.py +++ b/libs/aliyun/ecs.py @@ -1,279 +1,279 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# @Time : 2019/5/13 14:02 -# @Author : Fred Yangxiaofei -# @File : huawei_ecs.py -# @Role : 获取Aliyun资产信息推送到CMDB - - -import json -import re -from aliyunsdkcore.client import AcsClient -from aliyunsdkecs.request.v20140526 import DescribeInstancesRequest -from libs.common import M2human -from libs.db_context import DBContext -from models.server import Server, ServerDetail, AssetConfigs, model_to_dict -from libs.web_logs import ins_log -from opssdk.operate import MyCryptV2 -import fire - - -class EcsAPi(): - def __init__(self, access_id, access_key, region, default_admin_user): - self.access_id = access_id - self.access_key = access_key - self.region = region - self.default_admin_user = default_admin_user # 若用户给了默认管理用户，就给绑定上，else 为null - self.client = self.create_client() - self.page_number = 1 # 实例状态列表的页码。起始值：1 默认值：1 - self.page_size = 100 # 分页查询时设置的每页行数。最大值：100 默认值：10 - self.account = '阿里云' - self.state = 'auto' # auto状态：标记为自动获取的机器 - - def create_client(self): - client = AcsClient(self.access_id, self.access_key, self.region) - return client - - def set_request(self): - request = DescribeInstancesRequest.DescribeInstancesRequest() - request.set_PageNumber(self.page_number) - request.set_PageSize(self.page_size) - return request - - def get_response(self): - """ - 获取返回值 - :return: - """ - - response_data = {} - err = None - request = self.set_request() - try: - response = self.client.do_action_with_exception(request) - response_data = json.loads(str(response, encoding="utf8")) - except Exception as e: - err = e - return response_data, err - - def get_server_count(self): - """ - 获取机器总数 - :return: - """ - response_data, err = self.get_response() - if err != None: - ins_log.read_log('error', err) - return False - count = response_data['TotalCount'] - return count - - def get_server_info(self): - """ - 获取服务器信息 - :return: - """ - response_data, err = self.get_response() - if err != None: - ins_log.read_log('error', err) - return False - try: - ret = response_data['Instances']['Instance'] - except (KeyError, TypeError): - ins_log.read_log('error', '可能是因为SecretID/SecretKey配置错误，没法拿到配置，请检查下配置') - return False - server_list = [] - for i in ret: - asset_data = dict() - try: - asset_data['hostname'] = i.get('InstanceName') - except(KeyError, TypeError): - asset_data['hostname'] = i.get('InstanceId') # 取不到给instance_id - asset_data['region'] = i.get('ZoneId') - asset_data['instance_id'] = i.get('InstanceId') - asset_data['instance_type'] = i.get('InstanceType') - asset_data['instance_state'] = i.get('Status') - asset_data['cpu_cores'] = i.get('Cpu') - asset_data['memory'] = M2human(i.get('Memory')) - # 内网IP - try: - #VPC里面内网IP - asset_data['private_ip'] = i['VpcAttributes']['PrivateIpAddress']['IpAddress'][0] - except (KeyError, IndexError): - #非VPC里面获取内网IP - asset_data['private_ip'] = i['InnerIpAddress']['IpAddress'][0] - # 公网IP/弹性IP - try: - asset_data['public_ip'] = i['PublicIpAddress']['IpAddress'][0] - except(KeyError, IndexError): - asset_data['public_ip'] = i['EipAddress']['IpAddress'] - except Exception: - asset_data['public_ip'] = asset_data['private_ip'] - if 'public_ip' not in asset_data or not asset_data['public_ip'].strip(): - asset_data['public_ip'] = asset_data['private_ip'] - asset_data['os_type'] = i.get('OSType') - asset_data['os_name'] = i.get('OSName') - server_list.append(asset_data) - # print(asset_data) - ins_log.read_log('info', '资产信息:{}'.format(asset_data)) - return server_list - - def sync_cmdb(self): - """ - 写入CMDB - :return: - """ - - server_list = self.get_server_info() - if not server_list: - ins_log.read_log('info', 'Not fount server info') - return False - with DBContext('w') as session: - for server in server_list: - ip = server.get('public_ip') - private_ip = server.get('private_ip') - instance_id = server.get('instance_id', 'Null') - hostname = server.get('hostname', instance_id) - if hostname == '' or not hostname: - hostname = instance_id - - if re.search('syncserver', hostname): - hostname = '{}_{}'.format(hostname, private_ip) - - region = server.get('region', 'Null') - instance_type = server.get('instance_type', 'Null') - instance_state = server.get('instance_state', 'Null') - cpu = server.get('cpu', 'Null') - cpu_cores = server.get('cpu_cores', 'Null') - memory = server.get('memory', 'Null') - disk = server.get('disk', 'Null') # 阿里云接口里面好像没有disk信息 - os_type = server.get('os_type', 'Null') - os_name = server.get('os_name', 'Null') - os_kernel = server.get('os_kernel', 'Null') - sn = server.get('sn', 'Null') - - exist_ip_1 = session.query(Server).filter(Server.hostname == hostname).first() - if exist_ip_1: - session.query(Server).filter(Server.ip == ip).update( - {Server.hostname: hostname, Server.public_ip: ip, Server.private_ip: private_ip, - Server.idc: self.account, - Server.region: region, - Server.admin_user: self.default_admin_user}) - - else: - if os_type == 'windows': - # windows机器不绑定管理用户，资产信息只是平台拿到的一些基础信息 - new_windows_server = Server(ip=ip, public_ip=ip, private_ip=private_ip, hostname=hostname, - port=3389, idc=self.account, - region=region, - state=self.state) - session.add(new_windows_server) - - else: - # unix机器给默认绑定上管理用户，用于后续登陆机器拿详细资产使用的 - new_server = Server(ip=ip, public_ip=ip, private_ip=private_ip,hostname=hostname, port=54822, idc=self.account, - region=region, - state=self.state, admin_user=self.default_admin_user) - session.add(new_server) - - exist_ip = session.query(ServerDetail).filter(ServerDetail.ip == ip).first() - if exist_ip: - session.query(ServerDetail).filter(ServerDetail.ip == ip).update( - {ServerDetail.instance_id: instance_id, ServerDetail.instance_type: instance_type, - ServerDetail.instance_state: instance_state}) - else: - if os_type == 'windows': - - new_serve_detail = ServerDetail(ip=ip, instance_id=instance_id, instance_type=instance_type, - instance_state=instance_state, cpu=cpu, cpu_cores=cpu_cores, - memory=memory, os_type=os_name, disk=disk, os_kernel=os_kernel, - sn=sn) - session.add(new_serve_detail) - - else: - new_serve_detail = ServerDetail(ip=ip, instance_id=instance_id, instance_type=instance_type, - instance_state=instance_state, cpu=cpu, cpu_cores=cpu_cores, - memory=memory, os_type=os_name, disk=disk, os_kernel=os_kernel, - sn=sn) - session.add(new_serve_detail) - session.commit() - - def test_auth(self): - """ - 测试接口权限等信息是否异常 - :return: - """ - self.page_number = '1' - self.page_size = '1' - request = self.set_request() - response = self.client.do_action_with_exception(request) - response_data = json.loads(str(response, encoding="utf8")) - return response_data - - def index(self): - """ - 阿里云若机器超过100台需要进行通过PageSize+PageSize获取 - :return: - """ - - count = self.get_server_count() - print('Tocal：{}'.format(count)) - - - self.page_size = 100 - mod = count % self.page_size - if mod: - total_page_number = int(count / self.page_size) + 1 - else: - total_page_number = int(count / self.page_size) - for cur_page_number in range(1, total_page_number + 1): - self.page_number = cur_page_number - ins_log.read_log('info', '开始同步阿里云第{}页的{}台机器'.format(self.page_number, self.page_size)) - self.sync_cmdb() - - -def get_configs(): - """ - get id / key / region info - :return: - """ - - aliyun_configs_list = [] - with DBContext('r') as session: - aliyun_configs_info = session.query(AssetConfigs).filter(AssetConfigs.account == '阿里云', - AssetConfigs.state == 'true').all() - for data in aliyun_configs_info: - data_dict = model_to_dict(data) - data_dict['create_time'] = str(data_dict['create_time']) - data_dict['update_time'] = str(data_dict['update_time']) - aliyun_configs_list.append(data_dict) - return aliyun_configs_list - - -def main(): - """ - 从接口获取已经启用的配置 - :return: - """ - mc = MyCryptV2() - aliyun_configs_list = get_configs() - if not aliyun_configs_list: - ins_log.read_log('error', '没有获取到阿里云资产配置信息，跳过') - return False - for config in aliyun_configs_list: - access_id = config.get('access_id') - access_key = mc.my_decrypt(config.get('access_key')) # 解密后使用 - region = config.get('region') - default_admin_user = config.get('default_admin_user') - - obj = EcsAPi(access_id, access_key, region, default_admin_user) - obj.index() - - -# def test(): -# obj = EcsAPi('', '', 'cn-shanghai', '') -# obj.index() - - -if __name__ == '__main__': - fire.Fire(main) +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Time : 2019/5/13 14:02 +# @Author : Fred Yangxiaofei +# @File : huawei_ecs.py +# @Role : 获取Aliyun资产信息推送到CMDB + + +import json +import re +from aliyunsdkcore.client import AcsClient +from aliyunsdkecs.request.v20140526 import DescribeInstancesRequest +from libs.common import M2human +from libs.db_context import DBContext +from models.server import Server, ServerDetail, AssetConfigs, model_to_dict +from libs.web_logs import ins_log +from opssdk.operate import MyCryptV2 +import fire + + +class EcsAPi(): + def __init__(self, access_id, access_key, region, default_admin_user): + self.access_id = access_id + self.access_key = access_key + self.region = region + self.default_admin_user = default_admin_user # 若用户给了默认管理用户，就给绑定上，else 为null + self.client = self.create_client() + self.page_number = 1 # 实例状态列表的页码。起始值：1 默认值：1 + self.page_size = 100 # 分页查询时设置的每页行数。最大值：100 默认值：10 + self.account = '阿里云' + self.state = 'auto' # auto状态：标记为自动获取的机器 + + def create_client(self): + client = AcsClient(self.access_id, self.access_key, self.region) + return client + + def set_request(self): + request = DescribeInstancesRequest.DescribeInstancesRequest() + request.set_PageNumber(self.page_number) + request.set_PageSize(self.page_size) + return request + + def get_response(self): + """ + 获取返回值 + :return: + """ + + response_data = {} + err = None + request = self.set_request() + try: + response = self.client.do_action_with_exception(request) + response_data = json.loads(str(response, encoding="utf8")) + except Exception as e: + err = e + return response_data, err + + def get_server_count(self): + """ + 获取机器总数 + :return: + """ + response_data, err = self.get_response() + if err != None: + ins_log.read_log('error', err) + return False + count = response_data['TotalCount'] + return count + + def get_server_info(self): + """ + 获取服务器信息 + :return: + """ + response_data, err = self.get_response() + if err != None: + ins_log.read_log('error', err) + return False + try: + ret = response_data['Instances']['Instance'] + except (KeyError, TypeError): + ins_log.read_log('error', '可能是因为SecretID/SecretKey配置错误，没法拿到配置，请检查下配置') + return False + server_list = [] + for i in ret: + asset_data = dict() + try: + asset_data['hostname'] = i.get('InstanceName') + except(KeyError, TypeError): + asset_data['hostname'] = i.get('InstanceId') # 取不到给instance_id + asset_data['region'] = i.get('ZoneId') + asset_data['instance_id'] = i.get('InstanceId') + asset_data['instance_type'] = i.get('InstanceType') + asset_data['instance_state'] = i.get('Status') + asset_data['cpu_cores'] = i.get('Cpu') + asset_data['memory'] = M2human(i.get('Memory')) + # 内网IP + try: + #VPC里面内网IP + asset_data['private_ip'] = i['VpcAttributes']['PrivateIpAddress']['IpAddress'][0] + except (KeyError, IndexError): + #非VPC里面获取内网IP + asset_data['private_ip'] = i['InnerIpAddress']['IpAddress'][0] + # 公网IP/弹性IP + try: + asset_data['public_ip'] = i['PublicIpAddress']['IpAddress'][0] + except(KeyError, IndexError): + asset_data['public_ip'] = i['EipAddress']['IpAddress'] + except Exception: + asset_data['public_ip'] = asset_data['private_ip'] + if 'public_ip' not in asset_data or not asset_data['public_ip'].strip(): + asset_data['public_ip'] = asset_data['private_ip'] + asset_data['os_type'] = i.get('OSType') + asset_data['os_name'] = i.get('OSName') + server_list.append(asset_data) + # print(asset_data) + ins_log.read_log('info', '资产信息:{}'.format(asset_data)) + return server_list + + def sync_cmdb(self): + """ + 写入CMDB + :return: + """ + + server_list = self.get_server_info() + if not server_list: + ins_log.read_log('info', 'Not fount server info') + return False + with DBContext('w') as session: + for server in server_list: + ip = server.get('public_ip') + private_ip = server.get('private_ip') + instance_id = server.get('instance_id', 'Null') + hostname = server.get('hostname', instance_id) + if hostname == '' or not hostname: + hostname = instance_id + + if re.search('syncserver', hostname): + hostname = '{}_{}'.format(hostname, private_ip) + + region = server.get('region', 'Null') + instance_type = server.get('instance_type', 'Null') + instance_state = server.get('instance_state', 'Null') + cpu = server.get('cpu', 'Null') + cpu_cores = server.get('cpu_cores', 'Null') + memory = server.get('memory', 'Null') + disk = server.get('disk', 'Null') # 阿里云接口里面好像没有disk信息 + os_type = server.get('os_type', 'Null') + os_name = server.get('os_name', 'Null') + os_kernel = server.get('os_kernel', 'Null') + sn = server.get('sn', 'Null') + + exist_ip_1 = session.query(Server).filter(Server.hostname == hostname).first() + if exist_ip_1: + session.query(Server).filter(Server.ip == ip).update( + {Server.hostname: hostname, Server.public_ip: ip, Server.private_ip: private_ip, + Server.idc: self.account, + Server.region: region, + Server.admin_user: self.default_admin_user}) + + else: + if os_type == 'windows': + # windows机器不绑定管理用户，资产信息只是平台拿到的一些基础信息 + new_windows_server = Server(ip=ip, public_ip=ip, private_ip=private_ip, hostname=hostname, + port=3389, idc=self.account, + region=region, + state=self.state) + session.add(new_windows_server) + + else: + # unix机器给默认绑定上管理用户，用于后续登陆机器拿详细资产使用的 + new_server = Server(ip=ip, public_ip=ip, private_ip=private_ip,hostname=hostname, port=54822, idc=self.account, + region=region, + state=self.state, admin_user=self.default_admin_user) + session.add(new_server) + + exist_ip = session.query(ServerDetail).filter(ServerDetail.ip == ip).first() + if exist_ip: + session.query(ServerDetail).filter(ServerDetail.ip == ip).update( + {ServerDetail.instance_id: instance_id, ServerDetail.instance_type: instance_type, + ServerDetail.instance_state: instance_state}) + else: + if os_type == 'windows': + + new_serve_detail = ServerDetail(ip=ip, instance_id=instance_id, instance_type=instance_type, + instance_state=instance_state, cpu=cpu, cpu_cores=cpu_cores, + memory=memory, os_type=os_name, disk=disk, os_kernel=os_kernel, + sn=sn) + session.add(new_serve_detail) + + else: + new_serve_detail = ServerDetail(ip=ip, instance_id=instance_id, instance_type=instance_type, + instance_state=instance_state, cpu=cpu, cpu_cores=cpu_cores, + memory=memory, os_type=os_name, disk=disk, os_kernel=os_kernel, + sn=sn) + session.add(new_serve_detail) + session.commit() + + def test_auth(self): + """ + 测试接口权限等信息是否异常 + :return: + """ + self.page_number = '1' + self.page_size = '1' + request = self.set_request() + response = self.client.do_action_with_exception(request) + response_data = json.loads(str(response, encoding="utf8")) + return response_data + + def index(self): + """ + 阿里云若机器超过100台需要进行通过PageSize+PageSize获取 + :return: + """ + + count = self.get_server_count() + print('Tocal：{}'.format(count)) + + + self.page_size = 100 + mod = count % self.page_size + if mod: + total_page_number = int(count / self.page_size) + 1 + else: + total_page_number = int(count / self.page_size) + for cur_page_number in range(1, total_page_number + 1): + self.page_number = cur_page_number + ins_log.read_log('info', '开始同步阿里云第{}页的{}台机器'.format(self.page_number, self.page_size)) + self.sync_cmdb() + + +def get_configs(): + """ + get id / key / region info + :return: + """ + + aliyun_configs_list = [] + with DBContext('r') as session: + aliyun_configs_info = session.query(AssetConfigs).filter(AssetConfigs.account == '阿里云', + AssetConfigs.state == 'true').all() + for data in aliyun_configs_info: + data_dict = model_to_dict(data) + data_dict['create_time'] = str(data_dict['create_time']) + data_dict['update_time'] = str(data_dict['update_time']) + aliyun_configs_list.append(data_dict) + return aliyun_configs_list + + +def main(): + """ + 从接口获取已经启用的配置 + :return: + """ + mc = MyCryptV2() + aliyun_configs_list = get_configs() + if not aliyun_configs_list: + ins_log.read_log('error', '没有获取到阿里云资产配置信息，跳过') + return False + for config in aliyun_configs_list: + access_id = config.get('access_id') + access_key = mc.my_decrypt(config.get('access_key')) # 解密后使用 + region = config.get('region') + default_admin_user = config.get('default_admin_user') + + obj = EcsAPi(access_id, access_key, region, default_admin_user) + obj.index() + + +# def test(): +# obj = EcsAPi('', '', 'cn-shanghai', '') +# obj.index() + + +if __name__ == '__main__': + fire.Fire(main) diff --git a/libs/ansibleAPI/test02.py b/libs/ansibleAPI/test02.py index 911182d..13867e7 100644 --- a/libs/ansibleAPI/test02.py +++ b/libs/ansibleAPI/test02.py @@ -1,51 +1,51 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# @Time : 2019/5/1 13:17 -# @Author : Fred Yangxiaofei -# @File : test02.py.py -# @Role : 说明脚本功能 - - -from libs.ansibleAPI.runner import Runner -import json - - -def exec_shell(): - host_dict = { - "host": '172.16.0.101', - "port": '2222', - } - - - runner1 = Runner( - module_name="shell", - module_args="uptime", - remote_user="root", - pattern="all", - forks=5, - hosts= host_dict - ) - result1 = runner1.run() - print(result1) - if result1['dark']: - print(result1['dark']) - - - -def exec_setup(): - '''获取主机详情''' - runner2 = Runner( - module_name="setup", - module_args="", - remote_user="root", - pattern="all", - hosts="172.16.0.101" - ) - - result2 = runner2.run() - print(result2) - - -if __name__ == '__main__': - exec_shell() - # exec_setup() +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Time : 2019/5/1 13:17 +# @Author : Fred Yangxiaofei +# @File : test02.py.py +# @Role : 说明脚本功能 + + +from libs.ansibleAPI.runner import Runner +import json + + +def exec_shell(): + host_dict = { + "host": '172.16.0.101', + "port": '2222', + } + + + runner1 = Runner( + module_name="shell", + module_args="uptime", + remote_user="root", + pattern="all", + forks=5, + hosts= host_dict + ) + result1 = runner1.run() + print(result1) + if result1['dark']: + print(result1['dark']) + + + +def exec_setup(): + '''获取主机详情''' + runner2 = Runner( + module_name="setup", + module_args="", + remote_user="root", + pattern="all", + hosts="172.16.0.101" + ) + + result2 = runner2.run() + print(result2) + + +if __name__ == '__main__': + exec_shell() + # exec_setup() diff --git a/libs/aws/__init__.py b/libs/aws/__init__.py index fd0f105..92cfbfc 100644 --- a/libs/aws/__init__.py +++ b/libs/aws/__init__.py @@ -1,6 +1,6 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# @Time : 2019/5/13 16:35 -# @Author : Fred Yangxiaofei -# @File : __init__.py.py +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Time : 2019/5/13 16:35 +# @Author : Fred Yangxiaofei +# @File : __init__.py.py # @Role : 说明脚本功能 \ No newline at end of file diff --git a/libs/aws/ec2.py b/libs/aws/ec2.py index 1f6c0e3..e5ac1b6 100644 --- a/libs/aws/ec2.py +++ b/libs/aws/ec2.py @@ -1,194 +1,194 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# @Time : 2019/5/13 16:35 -# @Author : Fred Yangxiaofei -# @File : ec2.py -# @Role : 获取AWS Ec2信息推送到cmdb - - -import boto3 -import re -from libs.db_context import DBContext -from models.server import Server, ServerDetail, AssetConfigs, model_to_dict -from opssdk.operate import MyCryptV2 -from libs.web_logs import ins_log -import fire - - -class Ec2Api(): - def __init__(self, access_id, access_key, region, default_admin_user): - self.access_id = access_id - self.access_key = access_key - self.region = region - self.client = boto3.client('ec2', region_name=self.region, aws_access_key_id=self.access_id, - aws_secret_access_key=self.access_key) - self.default_admin_user = default_admin_user - self.account = 'AWS' # 标记AWS机器 - self.state = 'auto' # auto状态：标记为自动获取的机器 - - def get_response(self): - """ - 获取返回 - :return: - """ - try: - response = self.client.describe_instances() - return response - except Exception as e: - ins_log.read_log('error', e) - # print(e) - return False - - def get_server_info(self): - - response = self.get_response() - if not response: - ins_log.read_log('error', 'Not fount response, please check your access_id and access_key...') - # print('[Error]: Not fount response, please check your access_id and access_key...') - return False - - ret = response['Reservations'] - server_list = [] - if ret: - for r in ret: - for i in r['Instances']: - asset_data = dict() - try: - # asset_data['hostname'] = i.get('Tags')[0].get('Value') #这是旧的 - # AWS里面支持多个标签,我们只要Key为Name的 - tag_list = i.get('Tags') - # tag_list = [i for i in tag_list if i["Key"] == "Name"] - tag_list = filter(lambda x: x["Key"] == "Name", tag_list) - asset_data['hostname'] = list(tag_list)[0].get('Value') - - except (KeyError, TypeError, IndexError): - asset_data['hostname'] = i.get('InstanceId', 'Null') # 拿不到hostnameg给instance_id - - asset_data['region'] = i['Placement'].get('AvailabilityZone', 'Null') - asset_data['instance_id'] = i.get('InstanceId', 'Null') - asset_data['instance_type'] = i.get('InstanceType', 'Null') - asset_data['instance_state'] = i['State'].get('Name', '') - asset_data['private_ip'] = i.get('PrivateIpAddress', 'Null') - asset_data['public_ip'] = i.get('PublicIpAddress', asset_data['private_ip']) # 没有公网就给私网IP - # print(asset_data) - server_list.append(asset_data) - - return server_list - - def sync_cmdb(self): - """ - 写入CMDB - :return: - """ - - server_list = self.get_server_info() - if not server_list: - ins_log.read_log('info', 'Not fount server info...') - # print('Not Fount Server Info') - return False - with DBContext('w') as session: - for server in server_list: - ins_log.read_log('info', '资产信息:{}'.format(server)) - ip = server.get('public_ip') - private_ip = server.get('private_ip') - instance_id = server.get('instance_id', 'Null') - hostname = server.get('hostname', instance_id) - if not hostname.strip(): - hostname = instance_id - if re.search('syncserver', hostname): - hostname = '{}_{}'.format(hostname, private_ip) - - region = server.get('region', 'Null') - instance_type = server.get('instance_type', 'Null') - instance_state = server.get('instance_state', 'Null') - # AWS=接口没看到CPU这类信息 - cpu = server.get('cpu', 'Null') # CPU型号 - cpu_cores = server.get('cpu_cores', 'Null') - memory = server.get('memory', 'Null') - disk = server.get('disk', 'Null') - os_type = server.get('os_type', 'Null') - os_kernel = server.get('os_kernel', 'Null') - sn = server.get('sn', 'Null') - - exist_hostname = session.query(Server).filter(Server.hostname == hostname).first() - # exist_ip = session.query(Server).filter(Server.ip == ip).first() - if exist_hostname: - session.query(Server).filter(Server.hostname == hostname).update( - {Server.ip: ip, Server.public_ip: ip, Server.private_ip: private_ip, Server.idc: 'AWS', - Server.region: region}) - - else: - new_server = Server(ip=ip, public_ip=ip, private_ip=private_ip, hostname=hostname, port=22, - idc=self.account, - region=region, - state=self.state, admin_user=self.default_admin_user) - session.add(new_server) - - exist_ip = session.query(ServerDetail).filter(ServerDetail.ip == ip).first() - if exist_ip: - session.query(ServerDetail).filter(ServerDetail.ip == ip).update( - {ServerDetail.instance_id: instance_id, ServerDetail.instance_type: instance_type, - ServerDetail.instance_state: instance_state}) - else: - new_serve_detail = ServerDetail(ip=ip, instance_id=instance_id, instance_type=instance_type, - instance_state=instance_state, cpu=cpu, cpu_cores=cpu_cores, - memory=memory, disk=disk, os_type=os_type, os_kernel=os_kernel, - sn=sn) - session.add(new_serve_detail) - session.commit() - - def test_auth(self): - """ - 测试接口权限等信息是否异常 - :return: - """ - response = self.client.describe_instances() - return response - - -def get_configs(): - """ - get id / key / region info - :return: - """ - - aws_configs_list = [] - with DBContext('r') as session: - aws_configs_info = session.query(AssetConfigs).filter(AssetConfigs.account == 'AWS', - AssetConfigs.state == 'true').all() - for data in aws_configs_info: - data_dict = model_to_dict(data) - data_dict['create_time'] = str(data_dict['create_time']) - data_dict['update_time'] = str(data_dict['update_time']) - aws_configs_list.append(data_dict) - return aws_configs_list - - -def main(): - """ - 从接口获取已经启用的配置 - :return: - """ - - mc = MyCryptV2() - aws_configs_list = get_configs() - if not aws_configs_list: - ins_log.read_log('error', '没有获取到AWS资产配置信息，跳过') - return False - for config in aws_configs_list: - access_id = config.get('access_id') - access_key = mc.my_decrypt(config.get('access_key')) # 解密后使用 - region = config.get('region') - default_admin_user = config.get('default_admin_user') - - obj = Ec2Api(access_id, access_key, region, default_admin_user) - obj.sync_cmdb() - - -# def test(): -# obj = Ec2Api('','','us-east-1', '') -# obj.sync_cmdb() - - -if __name__ == '__main__': - fire.Fire(main) +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Time : 2019/5/13 16:35 +# @Author : Fred Yangxiaofei +# @File : ec2.py +# @Role : 获取AWS Ec2信息推送到cmdb + + +import boto3 +import re +from libs.db_context import DBContext +from models.server import Server, ServerDetail, AssetConfigs, model_to_dict +from opssdk.operate import MyCryptV2 +from libs.web_logs import ins_log +import fire + + +class Ec2Api(): + def __init__(self, access_id, access_key, region, default_admin_user): + self.access_id = access_id + self.access_key = access_key + self.region = region + self.client = boto3.client('ec2', region_name=self.region, aws_access_key_id=self.access_id, + aws_secret_access_key=self.access_key) + self.default_admin_user = default_admin_user + self.account = 'AWS' # 标记AWS机器 + self.state = 'auto' # auto状态：标记为自动获取的机器 + + def get_response(self): + """ + 获取返回 + :return: + """ + try: + response = self.client.describe_instances() + return response + except Exception as e: + ins_log.read_log('error', e) + # print(e) + return False + + def get_server_info(self): + + response = self.get_response() + if not response: + ins_log.read_log('error', 'Not fount response, please check your access_id and access_key...') + # print('[Error]: Not fount response, please check your access_id and access_key...') + return False + + ret = response['Reservations'] + server_list = [] + if ret: + for r in ret: + for i in r['Instances']: + asset_data = dict() + try: + # asset_data['hostname'] = i.get('Tags')[0].get('Value') #这是旧的 + # AWS里面支持多个标签,我们只要Key为Name的 + tag_list = i.get('Tags') + # tag_list = [i for i in tag_list if i["Key"] == "Name"] + tag_list = filter(lambda x: x["Key"] == "Name", tag_list) + asset_data['hostname'] = list(tag_list)[0].get('Value') + + except (KeyError, TypeError, IndexError): + asset_data['hostname'] = i.get('InstanceId', 'Null') # 拿不到hostnameg给instance_id + + asset_data['region'] = i['Placement'].get('AvailabilityZone', 'Null') + asset_data['instance_id'] = i.get('InstanceId', 'Null') + asset_data['instance_type'] = i.get('InstanceType', 'Null') + asset_data['instance_state'] = i['State'].get('Name', '') + asset_data['private_ip'] = i.get('PrivateIpAddress', 'Null') + asset_data['public_ip'] = i.get('PublicIpAddress', asset_data['private_ip']) # 没有公网就给私网IP + # print(asset_data) + server_list.append(asset_data) + + return server_list + + def sync_cmdb(self): + """ + 写入CMDB + :return: + """ + + server_list = self.get_server_info() + if not server_list: + ins_log.read_log('info', 'Not fount server info...') + # print('Not Fount Server Info') + return False + with DBContext('w') as session: + for server in server_list: + ins_log.read_log('info', '资产信息:{}'.format(server)) + ip = server.get('public_ip') + private_ip = server.get('private_ip') + instance_id = server.get('instance_id', 'Null') + hostname = server.get('hostname', instance_id) + if not hostname.strip(): + hostname = instance_id + if re.search('syncserver', hostname): + hostname = '{}_{}'.format(hostname, private_ip) + + region = server.get('region', 'Null') + instance_type = server.get('instance_type', 'Null') + instance_state = server.get('instance_state', 'Null') + # AWS=接口没看到CPU这类信息 + cpu = server.get('cpu', 'Null') # CPU型号 + cpu_cores = server.get('cpu_cores', 'Null') + memory = server.get('memory', 'Null') + disk = server.get('disk', 'Null') + os_type = server.get('os_type', 'Null') + os_kernel = server.get('os_kernel', 'Null') + sn = server.get('sn', 'Null') + + exist_hostname = session.query(Server).filter(Server.hostname == hostname).first() + # exist_ip = session.query(Server).filter(Server.ip == ip).first() + if exist_hostname: + session.query(Server).filter(Server.hostname == hostname).update( + {Server.ip: ip, Server.public_ip: ip, Server.private_ip: private_ip, Server.idc: 'AWS', + Server.region: region}) + + else: + new_server = Server(ip=ip, public_ip=ip, private_ip=private_ip, hostname=hostname, port=22, + idc=self.account, + region=region, + state=self.state, admin_user=self.default_admin_user) + session.add(new_server) + + exist_ip = session.query(ServerDetail).filter(ServerDetail.ip == ip).first() + if exist_ip: + session.query(ServerDetail).filter(ServerDetail.ip == ip).update( + {ServerDetail.instance_id: instance_id, ServerDetail.instance_type: instance_type, + ServerDetail.instance_state: instance_state}) + else: + new_serve_detail = ServerDetail(ip=ip, instance_id=instance_id, instance_type=instance_type, + instance_state=instance_state, cpu=cpu, cpu_cores=cpu_cores, + memory=memory, disk=disk, os_type=os_type, os_kernel=os_kernel, + sn=sn) + session.add(new_serve_detail) + session.commit() + + def test_auth(self): + """ + 测试接口权限等信息是否异常 + :return: + """ + response = self.client.describe_instances() + return response + + +def get_configs(): + """ + get id / key / region info + :return: + """ + + aws_configs_list = [] + with DBContext('r') as session: + aws_configs_info = session.query(AssetConfigs).filter(AssetConfigs.account == 'AWS', + AssetConfigs.state == 'true').all() + for data in aws_configs_info: + data_dict = model_to_dict(data) + data_dict['create_time'] = str(data_dict['create_time']) + data_dict['update_time'] = str(data_dict['update_time']) + aws_configs_list.append(data_dict) + return aws_configs_list + + +def main(): + """ + 从接口获取已经启用的配置 + :return: + """ + + mc = MyCryptV2() + aws_configs_list = get_configs() + if not aws_configs_list: + ins_log.read_log('error', '没有获取到AWS资产配置信息，跳过') + return False + for config in aws_configs_list: + access_id = config.get('access_id') + access_key = mc.my_decrypt(config.get('access_key')) # 解密后使用 + region = config.get('region') + default_admin_user = config.get('default_admin_user') + + obj = Ec2Api(access_id, access_key, region, default_admin_user) + obj.sync_cmdb() + + +# def test(): +# obj = Ec2Api('','','us-east-1', '') +# obj.sync_cmdb() + + +if __name__ == '__main__': + fire.Fire(main) diff --git a/libs/common.py b/libs/common.py index 4890692..88cb328 100644 --- a/libs/common.py +++ b/libs/common.py @@ -1,158 +1,158 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# @Time : 2019/4/22 18:53 -# @Author : Fred Yangxiaofei -# @File : common.py -# @Role : 公用方法 - -import sys -import time -import subprocess -import paramiko -import concurrent.futures -import re -from libs.ansibleAPI.runner import Runner - - -def is_number(s): - try: - float(s) - return True - except ValueError: - pass - - try: - import unicodedata - unicodedata.numeric(s) - return True - except (TypeError, ValueError): - pass - - return False - -def color_print(msg, color='red', exits=False): - '''颜色选择， 判断退出''' - color_msg = {'blue': '\033[1;36m%s\033[0m', - 'green': '\033[1;32m%s\033[0m', - 'yellow': '\033[1;33m%s\033[0m', - 'red': '\033[1;31m%s\033[0m', - 'title': '\033[30;42m%s\033[0m', - 'info': '\033[32m%s\033[0m'} - msg = color_msg.get(color, 'red') % msg - print(msg) - if exits: - time.sleep(2) - sys.exit() - return msg - - - -def M2human(n): - symbols = ('G', 'T', 'P', 'E', 'Z', 'Y') - prefix = {} - for i, s in enumerate(symbols): - prefix[s] = 1 << (i + 1) * 10 - for s in reversed(symbols): - if n >= prefix[s]: - value = float(n) / prefix[s] - return '%.1f%s' % (value, s) - return '%sM' % n - -def exec_setup(user, host): - '''获取主机详情''' - runner = Runner( - module_name="setup", - module_args="", - remote_user=user, - pattern="all", - hosts=host - ) - - result = runner.run() - print(result) - if result['dark']: - return False - else: - return result - - -def exec_shell(cmd): - '''执行shell命令函数''' - sub2 = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - stdout, stderr = sub2.communicate() - ret = sub2.returncode - return ret,stdout.decode('utf-8').strip() - - -def exec_thread(func, iterable1): - ### 取所有主机 最多启动100个进程 - pool_num = 10 - with concurrent.futures.ProcessPoolExecutor(pool_num) as executor: - results = executor.map(func, iterable1) - return results - - -def get_key_file(ssh_key): - '''根据key内容临时生成一个key文件''' - #file_path = '/tmp/%s'%shortuuid.uuid() - file_path = '/tmp/codo_cmdb_id_rsa' - cmd = 'echo "{}" > {} && chmod 600 {}'.format(ssh_key,file_path,file_path) - ret,stdout = exec_shell(cmd) - if ret == 0: - return file_path - else: - return None - - -# def remote_upload_file(ip,user,pwd,cmd,local_file,remote_file,port=22): -# '''ssh上传并执行bash for pwd''' -# t = paramiko.Transport((ip,port)) -# t.connect(username=user, password=pwd) -# sftp = paramiko.SFTPClient.from_transport(t) -# sftp.put(local_file,remote_file) -# ssh = paramiko.SSHClient() -# ssh._transport = t -# stdin, stdout, stderr = ssh.exec_command(cmd) -# show_log = stdout.read().decode('utf-8').strip() -# t.close() -# return show_log - -def remote_upload_file(ip,user,ssh_key,cmd,local_file,remote_file,port=22): - '''ssh上传并执行bash for key''' - private_key = paramiko.RSAKey.from_private_key_file(ssh_key) - ssh = paramiko.SSHClient() - ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - ssh.connect(hostname=ip,port=port,username=user, pkey=private_key, timeout=10) - sftp = ssh.open_sftp() - sftp.put(local_file, remote_file) - sftp.close() - - stdin, stdout, stderr = ssh.exec_command(cmd) - show_log = stdout.read().decode('utf-8').strip() - err_log = stderr.read().decode('utf-8').strip() - #print(show_log,err_log) - ssh.close() - return show_log, err_log - - - -def remote_exec_cmd(ip,user,ssh_key,cmd,port=22): - '''ssh上传并执行bash for key''' - private_key = paramiko.RSAKey.from_private_key_file(ssh_key) - ssh = paramiko.SSHClient() - ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - ssh.connect(hostname=ip,port=port,username=user, pkey=private_key, timeout=10) - stdin, stdout, stderr = ssh.exec_command(cmd) - show_log = stdout.read().decode('utf-8').strip() - err_log = stderr.read().decode('utf-8').strip() - #print(show_log,err_log) - ssh.close() - return show_log, err_log - - -def check_ip(ip_address): - compile_ip=re.compile('^(1\d{2}|2[0-4]\d|25[0-5]|[1-9]\d|[1-9])\.(1\d{2}|2[0-4]\d|25[0-5]|[1-9]\d|\d)\.(1\d{2}|2[0-4]\d|25[0-5]|[1-9]\d|\d)\.(1\d{2}|2[0-4]\d|25[0-5]|[1-9]\d|\d)$') - if compile_ip.match(ip_address): - return True - else: +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Time : 2019/4/22 18:53 +# @Author : Fred Yangxiaofei +# @File : common.py +# @Role : 公用方法 + +import sys +import time +import subprocess +import paramiko +import concurrent.futures +import re +from libs.ansibleAPI.runner import Runner + + +def is_number(s): + try: + float(s) + return True + except ValueError: + pass + + try: + import unicodedata + unicodedata.numeric(s) + return True + except (TypeError, ValueError): + pass + + return False + +def color_print(msg, color='red', exits=False): + '''颜色选择， 判断退出''' + color_msg = {'blue': '\033[1;36m%s\033[0m', + 'green': '\033[1;32m%s\033[0m', + 'yellow': '\033[1;33m%s\033[0m', + 'red': '\033[1;31m%s\033[0m', + 'title': '\033[30;42m%s\033[0m', + 'info': '\033[32m%s\033[0m'} + msg = color_msg.get(color, 'red') % msg + print(msg) + if exits: + time.sleep(2) + sys.exit() + return msg + + + +def M2human(n): + symbols = ('G', 'T', 'P', 'E', 'Z', 'Y') + prefix = {} + for i, s in enumerate(symbols): + prefix[s] = 1 << (i + 1) * 10 + for s in reversed(symbols): + if n >= prefix[s]: + value = float(n) / prefix[s] + return '%.1f%s' % (value, s) + return '%sM' % n + +def exec_setup(user, host): + '''获取主机详情''' + runner = Runner( + module_name="setup", + module_args="", + remote_user=user, + pattern="all", + hosts=host + ) + + result = runner.run() + print(result) + if result['dark']: + return False + else: + return result + + +def exec_shell(cmd): + '''执行shell命令函数''' + sub2 = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + stdout, stderr = sub2.communicate() + ret = sub2.returncode + return ret,stdout.decode('utf-8').strip() + + +def exec_thread(func, iterable1): + ### 取所有主机 最多启动100个进程 + pool_num = 10 + with concurrent.futures.ProcessPoolExecutor(pool_num) as executor: + results = executor.map(func, iterable1) + return results + + +def get_key_file(ssh_key): + '''根据key内容临时生成一个key文件''' + #file_path = '/tmp/%s'%shortuuid.uuid() + file_path = '/tmp/codo_cmdb_id_rsa' + cmd = 'echo "{}" > {} && chmod 600 {}'.format(ssh_key,file_path,file_path) + ret,stdout = exec_shell(cmd) + if ret == 0: + return file_path + else: + return None + + +# def remote_upload_file(ip,user,pwd,cmd,local_file,remote_file,port=22): +# '''ssh上传并执行bash for pwd''' +# t = paramiko.Transport((ip,port)) +# t.connect(username=user, password=pwd) +# sftp = paramiko.SFTPClient.from_transport(t) +# sftp.put(local_file,remote_file) +# ssh = paramiko.SSHClient() +# ssh._transport = t +# stdin, stdout, stderr = ssh.exec_command(cmd) +# show_log = stdout.read().decode('utf-8').strip() +# t.close() +# return show_log + +def remote_upload_file(ip,user,ssh_key,cmd,local_file,remote_file,port=22): + '''ssh上传并执行bash for key''' + private_key = paramiko.RSAKey.from_private_key_file(ssh_key) + ssh = paramiko.SSHClient() + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + ssh.connect(hostname=ip,port=port,username=user, pkey=private_key, timeout=10) + sftp = ssh.open_sftp() + sftp.put(local_file, remote_file) + sftp.close() + + stdin, stdout, stderr = ssh.exec_command(cmd) + show_log = stdout.read().decode('utf-8').strip() + err_log = stderr.read().decode('utf-8').strip() + #print(show_log,err_log) + ssh.close() + return show_log, err_log + + + +def remote_exec_cmd(ip,user,ssh_key,cmd,port=22): + '''ssh上传并执行bash for key''' + private_key = paramiko.RSAKey.from_private_key_file(ssh_key) + ssh = paramiko.SSHClient() + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + ssh.connect(hostname=ip,port=port,username=user, pkey=private_key, timeout=10) + stdin, stdout, stderr = ssh.exec_command(cmd) + show_log = stdout.read().decode('utf-8').strip() + err_log = stderr.read().decode('utf-8').strip() + #print(show_log,err_log) + ssh.close() + return show_log, err_log + + +def check_ip(ip_address): + compile_ip=re.compile('^(1\d{2}|2[0-4]\d|25[0-5]|[1-9]\d|[1-9])\.(1\d{2}|2[0-4]\d|25[0-5]|[1-9]\d|\d)\.(1\d{2}|2[0-4]\d|25[0-5]|[1-9]\d|\d)\.(1\d{2}|2[0-4]\d|25[0-5]|[1-9]\d|\d)$') + if compile_ip.match(ip_address): + return True + else: return False \ No newline at end of file diff --git a/libs/huaweiyun/__init__.py b/libs/huaweiyun/__init__.py index 31104af..a5d1d6a 100644 --- a/libs/huaweiyun/__init__.py +++ b/libs/huaweiyun/__init__.py @@ -1,6 +1,6 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# @Time : 2019/6/18 9:30 -# @Author : Fred Yangxiaofei -# @File : __init__.py.py +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Time : 2019/6/18 9:30 +# @Author : Fred Yangxiaofei +# @File : __init__.py.py # @Role : 说明脚本功能 \ No newline at end of file diff --git a/libs/huaweiyun/huawei_ecs.py b/libs/huaweiyun/huawei_ecs.py index f44ef9a..b77d0a9 100644 --- a/libs/huaweiyun/huawei_ecs.py +++ b/libs/huaweiyun/huawei_ecs.py @@ -1,196 +1,196 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# @Time : 2019/6/18 9:31 -# @Author : Fred Yangxiaofei -# @File : huawei_ecs.py -# @Role : Huawei Cloud Ecs - - -from openstack import connection -from libs.web_logs import ins_log -from libs.db_context import DBContext -from models.server import Server, ServerDetail, AssetConfigs, model_to_dict -from opssdk.operate import MyCryptV2 -import fire - - -class HuaweiEcsApi(): - def __init__(self, access_id, access_key, region, cloud, project_id, default_admin_user): - """ - - :param access_id: AccessID - :param access_key: Accesskey - :param region: 区域，如：cn-east-2 - :param cloud: 默认：myhuaweicloud.com # cdn use: cloud = "myhwclouds.com" - :param project_id: 这个字面是项目ID，其实就是华为云，我的凭证--项目ID，这是对应区域的，每个区域都有一个 - """ - self.default_admin_user = default_admin_user - self.account = '华为云' # 标记华为云机器 - self.state = 'auto' # auto状态：标记为自动获取的机器 - self.access_id = access_id - self.access_key = access_key - self.region = region - self.cloud = cloud - self.project_id = project_id - self.conn = self.connect() - - def connect(self): - try: - conn = connection.Connection( - project_id=self.project_id, - cloud=self.cloud, - region=self.region, - ak=self.access_id, - sk=self.access_key) - return conn - except Exception as e: - print(e) - return False - - def get_server_info(self): - """ - 这里吐槽一下，华为云请求返回过来给我一个class 数据居然没处理/// 也可能是因为我用法不对，But 官方提供就这么用的, 所以我这里自己处理了 - :return: - """ - - servers = self.conn.compute.servers(limit=10) # 一次取10台,迭代取 - server_list = [] - for server in servers: - asset_data = dict() - asset_data['hostname'] = server.name - ip_info = server.addresses - for k, v in ip_info.items(): - for i in v: - # 这是弹性IP - if 'floating' in i.values(): - asset_data['public_ip'] = i.get('addr') - else: - asset_data['public_ip'] = i.get('addr') - asset_data['instance_type'] = server.flavor.get('id') - asset_data['instance_id'] = server.id - asset_data['instance_status'] = server.status - asset_data['region'] = server.availability_zone - server_list.append(asset_data) - return server_list - - def sync_cmdb(self): - """ - 数据写CMDB，华为的信息比较少 - :return: - """ - server_list = self.get_server_info() - if not server_list: - ins_log.read_log('info', 'Not fount server info...') - return False - with DBContext('w') as session: - for server in server_list: - print(server) - ip = server.get('public_ip') - instance_id = server.get('instance_id', 'Null') - hostname = server.get('hostname', instance_id) - if not hostname.strip(): - hostname = instance_id - region = server.get('region', 'Null') - instance_type = server.get('instance_type', 'Null') - instance_status = server.get('instance_status') - # 华为云接口没看到CPU这类信息 - cpu = server.get('cpu', 'Null') # CPU型号 - cpu_cores = server.get('cpu_cores', 'Null') - memory = server.get('memory', 'Null') - disk = server.get('disk', 'Null') - os_type = server.get('os_type', 'Null') - os_kernel = server.get('os_kernel', 'Null') - sn = server.get('sn', 'Null') - exist_hostname = session.query(Server).filter(Server.hostname == hostname).first() - # exist_ip = session.query(Server).filter(Server.ip == ip).first() - if exist_hostname: - session.query(Server).filter(Server.hostname == hostname).update( - {Server.ip: ip, Server.public_ip: ip, Server.idc: self.account, Server.region: region}) - - else: - new_server = Server(ip=ip, public_ip=ip, hostname=hostname, port=22, idc=self.account, - region=region, - state=self.state, admin_user=self.default_admin_user) - session.add(new_server) - - exist_ip = session.query(ServerDetail).filter(ServerDetail.ip == ip).first() - if exist_ip: - session.query(ServerDetail).filter(ServerDetail.ip == ip).update( - {ServerDetail.instance_id: instance_id, ServerDetail.instance_type: instance_type, - ServerDetail.instance_state: instance_status}) - else: - new_serve_detail = ServerDetail(ip=ip, instance_id=instance_id, instance_type=instance_type, - instance_state=instance_status, cpu=cpu, cpu_cores=cpu_cores, - memory=memory, disk=disk, os_type=os_type, os_kernel=os_kernel, - sn=sn) - session.add(new_serve_detail) - session.commit() - - def test_auth(self, huawei_instance_id): - """ - 实在无能为力，华为API文档太少，没有深入研究，测试查询一台实例ID的信息测试 - :return: - """ - server_msg = self.conn.compute.get_server(huawei_instance_id) - return server_msg - # try: - # self.conn.compute.get_server(huawei_instance_id) - # msg = '成功' - # except keystoneauth1.exceptions.http.Unauthorized: - # print('请检查AccessID和AccessKey权限是否正确') - # msg = '请检查AccessID和AccessKey权限是否正确' - # except openstack.exceptions.HttpException as err: - # msg = 'openstack error for {}'.format(err) - # print('openstack error') - # print(err) - # except Exception as e: - # print(e) - # msg = 'error: {}'.format(e) - # return msg - - -def get_configs(): - """ - get id / key / region info - :return: - """ - - huawei_configs_list = [] - with DBContext('r') as session: - huawei_configs_info = session.query(AssetConfigs).filter(AssetConfigs.account == '华为云', - AssetConfigs.state == 'true').all() - for data in huawei_configs_info: - data_dict = model_to_dict(data) - data_dict['create_time'] = str(data_dict['create_time']) - data_dict['update_time'] = str(data_dict['update_time']) - huawei_configs_list.append(data_dict) - return huawei_configs_list - - -def main(): - """ - 从接口获取已经启用的配置 - :return: - """ - - mc = MyCryptV2() - huawei_configs_list = get_configs() - if not huawei_configs_list: - ins_log.read_log('error', '没有获取到华为云资产配置信息，跳过') - return False - for config in huawei_configs_list: - access_id = config.get('access_id') - access_key = mc.my_decrypt(config.get('access_key')) # 解密后使用 - region = config.get('region') - huawei_cloud = config.get('huawei_cloud') - project_id = config.get('project_id') - - default_admin_user = config.get('default_admin_user') - obj = HuaweiEcsApi(access_id=access_id, access_key=access_key, region=region, cloud=huawei_cloud, - project_id=project_id, - default_admin_user=default_admin_user) - obj.sync_cmdb() - - -if __name__ == '__main__': - fire.Fire(main) +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Time : 2019/6/18 9:31 +# @Author : Fred Yangxiaofei +# @File : huawei_ecs.py +# @Role : Huawei Cloud Ecs + + +from openstack import connection +from libs.web_logs import ins_log +from libs.db_context import DBContext +from models.server import Server, ServerDetail, AssetConfigs, model_to_dict +from opssdk.operate import MyCryptV2 +import fire + + +class HuaweiEcsApi(): + def __init__(self, access_id, access_key, region, cloud, project_id, default_admin_user): + """ + + :param access_id: AccessID + :param access_key: Accesskey + :param region: 区域，如：cn-east-2 + :param cloud: 默认：myhuaweicloud.com # cdn use: cloud = "myhwclouds.com" + :param project_id: 这个字面是项目ID，其实就是华为云，我的凭证--项目ID，这是对应区域的，每个区域都有一个 + """ + self.default_admin_user = default_admin_user + self.account = '华为云' # 标记华为云机器 + self.state = 'auto' # auto状态：标记为自动获取的机器 + self.access_id = access_id + self.access_key = access_key + self.region = region + self.cloud = cloud + self.project_id = project_id + self.conn = self.connect() + + def connect(self): + try: + conn = connection.Connection( + project_id=self.project_id, + cloud=self.cloud, + region=self.region, + ak=self.access_id, + sk=self.access_key) + return conn + except Exception as e: + print(e) + return False + + def get_server_info(self): + """ + 这里吐槽一下，华为云请求返回过来给我一个class 数据居然没处理/// 也可能是因为我用法不对，But 官方提供就这么用的, 所以我这里自己处理了 + :return: + """ + + servers = self.conn.compute.servers(limit=10) # 一次取10台,迭代取 + server_list = [] + for server in servers: + asset_data = dict() + asset_data['hostname'] = server.name + ip_info = server.addresses + for k, v in ip_info.items(): + for i in v: + # 这是弹性IP + if 'floating' in i.values(): + asset_data['public_ip'] = i.get('addr') + else: + asset_data['public_ip'] = i.get('addr') + asset_data['instance_type'] = server.flavor.get('id') + asset_data['instance_id'] = server.id + asset_data['instance_status'] = server.status + asset_data['region'] = server.availability_zone + server_list.append(asset_data) + return server_list + + def sync_cmdb(self): + """ + 数据写CMDB，华为的信息比较少 + :return: + """ + server_list = self.get_server_info() + if not server_list: + ins_log.read_log('info', 'Not fount server info...') + return False + with DBContext('w') as session: + for server in server_list: + print(server) + ip = server.get('public_ip') + instance_id = server.get('instance_id', 'Null') + hostname = server.get('hostname', instance_id) + if not hostname.strip(): + hostname = instance_id + region = server.get('region', 'Null') + instance_type = server.get('instance_type', 'Null') + instance_status = server.get('instance_status') + # 华为云接口没看到CPU这类信息 + cpu = server.get('cpu', 'Null') # CPU型号 + cpu_cores = server.get('cpu_cores', 'Null') + memory = server.get('memory', 'Null') + disk = server.get('disk', 'Null') + os_type = server.get('os_type', 'Null') + os_kernel = server.get('os_kernel', 'Null') + sn = server.get('sn', 'Null') + exist_hostname = session.query(Server).filter(Server.hostname == hostname).first() + # exist_ip = session.query(Server).filter(Server.ip == ip).first() + if exist_hostname: + session.query(Server).filter(Server.hostname == hostname).update( + {Server.ip: ip, Server.public_ip: ip, Server.idc: self.account, Server.region: region}) + + else: + new_server = Server(ip=ip, public_ip=ip, hostname=hostname, port=22, idc=self.account, + region=region, + state=self.state, admin_user=self.default_admin_user) + session.add(new_server) + + exist_ip = session.query(ServerDetail).filter(ServerDetail.ip == ip).first() + if exist_ip: + session.query(ServerDetail).filter(ServerDetail.ip == ip).update( + {ServerDetail.instance_id: instance_id, ServerDetail.instance_type: instance_type, + ServerDetail.instance_state: instance_status}) + else: + new_serve_detail = ServerDetail(ip=ip, instance_id=instance_id, instance_type=instance_type, + instance_state=instance_status, cpu=cpu, cpu_cores=cpu_cores, + memory=memory, disk=disk, os_type=os_type, os_kernel=os_kernel, + sn=sn) + session.add(new_serve_detail) + session.commit() + + def test_auth(self, huawei_instance_id): + """ + 实在无能为力，华为API文档太少，没有深入研究，测试查询一台实例ID的信息测试 + :return: + """ + server_msg = self.conn.compute.get_server(huawei_instance_id) + return server_msg + # try: + # self.conn.compute.get_server(huawei_instance_id) + # msg = '成功' + # except keystoneauth1.exceptions.http.Unauthorized: + # print('请检查AccessID和AccessKey权限是否正确') + # msg = '请检查AccessID和AccessKey权限是否正确' + # except openstack.exceptions.HttpException as err: + # msg = 'openstack error for {}'.format(err) + # print('openstack error') + # print(err) + # except Exception as e: + # print(e) + # msg = 'error: {}'.format(e) + # return msg + + +def get_configs(): + """ + get id / key / region info + :return: + """ + + huawei_configs_list = [] + with DBContext('r') as session: + huawei_configs_info = session.query(AssetConfigs).filter(AssetConfigs.account == '华为云', + AssetConfigs.state == 'true').all() + for data in huawei_configs_info: + data_dict = model_to_dict(data) + data_dict['create_time'] = str(data_dict['create_time']) + data_dict['update_time'] = str(data_dict['update_time']) + huawei_configs_list.append(data_dict) + return huawei_configs_list + + +def main(): + """ + 从接口获取已经启用的配置 + :return: + """ + + mc = MyCryptV2() + huawei_configs_list = get_configs() + if not huawei_configs_list: + ins_log.read_log('error', '没有获取到华为云资产配置信息，跳过') + return False + for config in huawei_configs_list: + access_id = config.get('access_id') + access_key = mc.my_decrypt(config.get('access_key')) # 解密后使用 + region = config.get('region') + huawei_cloud = config.get('huawei_cloud') + project_id = config.get('project_id') + + default_admin_user = config.get('default_admin_user') + obj = HuaweiEcsApi(access_id=access_id, access_key=access_key, region=region, cloud=huawei_cloud, + project_id=project_id, + default_admin_user=default_admin_user) + obj.sync_cmdb() + + +if __name__ == '__main__': + fire.Fire(main) diff --git a/libs/qcloud/__init__.py b/libs/qcloud/__init__.py index e29464a..59f3e9a 100644 --- a/libs/qcloud/__init__.py +++ b/libs/qcloud/__init__.py @@ -1,6 +1,6 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# @Time : 2019/5/10 14:56 -# @Author : Fred Yangxiaofei -# @File : __init__.py.py +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Time : 2019/5/10 14:56 +# @Author : Fred Yangxiaofei +# @File : __init__.py.py # @Role : 说明脚本功能 \ No newline at end of file diff --git a/libs/qcloud/cvm.py b/libs/qcloud/cvm.py index 94126c8..30170d7 100644 --- a/libs/qcloud/cvm.py +++ b/libs/qcloud/cvm.py @@ -1,259 +1,259 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# @Time : 2019/5/10 16:37 -# @Author : Fred Yangxiaofei -# @File : cvm.py -# @Role : 获取腾讯云主机信息 - - -import time -import random -import requests -import json -import re -from libs.qcloud.qcloud_api import ApiOper -from libs.db_context import DBContext -from libs.web_logs import ins_log -from opssdk.operate import MyCryptV2 -from models.server import Server, ServerDetail, AssetConfigs, model_to_dict -import fire - - -class CVMApi(): - def __init__(self, access_id, access_key, region, default_admin_user): - self.offset = '0' # 偏移量,这里拼接的时候必须是字符串 - self.limit = '100' # 官方默认是20，大于100需要设置偏移量再次请求：offset=100,offset={机器总数} - self.access_id = access_id - self.access_key = access_key - self.region = region - self.default_admin_user = default_admin_user # 默认管理用户 - self.account = '腾讯云' # 标记机器为腾讯云 - self.state = 'auto' # 标记自动获取状态 - - def get_server_url(self): - """ - 获取腾讯云机器的API拼接结果，get请求 - :return: - """ - api_url = 'cvm.tencentcloudapi.com/?' - keydict = { - # 公共参数部分 - 'Timestamp': str(int(time.time())), - 'Nonce': str(int(random.random() * 1000)), - 'Region': self.region, - 'SecretId': self.access_id, - 'Version': '2017-03-12', - # 'SignatureMethod': SignatureMethod, - # 接口参数部分 - 'Action': 'DescribeInstances', - 'Offset': self.offset, # 机器数量超过100需要用到偏移量 - 'Limit': self.limit, - } - result_url = ApiOper.run(keydict, api_url, self.access_key) - return result_url - - def get_result_data(self): - """ - 获取返回的数据 - :return: - """ - result_url = self.get_server_url() - response = requests.get(result_url) - result_data = json.loads(response.text) - if result_data['Response'].get('Error'): - ins_log.read_log('error', '{}'.format(result_data['Response'])) - return False - else: - ret = result_data['Response'] - - return ret - - def get_server_count(self): - """ - 获取主机总数量。如果机器数量大于limit限制就要用到offset偏移查询 - :return: int - """ - - ret = self.get_result_data() - if not ret: - return False - - server_count = ret['TotalCount'] - return server_count - - def get_server_info(self): - """ - 获取QCloud服务器信息 - :return: - """ - - ret = self.get_result_data() - if not ret: - return False - - server_list = [] - - server_info = ret['InstanceSet'] - for i in server_info: - asset_data = dict() - instance_id = i.get('InstanceId') - instance_state = i.get('InstanceState') - instance_type = i.get('InstanceType') - cpu = i.get('CPU') - memory = i.get('Memory') - hostname = i.get('InstanceName') - disk = i['SystemDisk'].get('DiskSize') - try: - private_ip = i['PrivateIpAddresses'][0] - except (KeyError, TypeError): - private_ip = 'Null' - try: - public_ip = i['PublicIpAddresses'][0] - except (KeyError, TypeError): - public_ip = private_ip # 不存在公网就给私网IP - os_type = i.get('OsName') - region = i['Placement'].get('Zone') - asset_data['region'] = region - asset_data['instance_id'] = instance_id - asset_data['instance_state'] = instance_state - asset_data['instance_type'] = instance_type - asset_data['cpu_cores'] = '{}Core'.format(cpu) - asset_data['memory'] = '{}G'.format(memory) - asset_data['hostname'] = hostname - asset_data['disk'] = '{}G'.format(disk) - asset_data['private_ip'] = private_ip - asset_data['public_ip'] = public_ip - asset_data['os_type'] = os_type - # print(asset_data) - server_list.append(asset_data) - ins_log.read_log('info', '资产信息:{}'.format(asset_data)) - return server_list - - def sync_cmdb(self): - """ - 返回结果写入CMDB - :return: - """ - server_list = self.get_server_info() - - if not server_list: - print('Not Fount Server Info') - return False - - with DBContext('w') as session: - for server in server_list: - ip = server.get('public_ip') - private_ip = server.get('private_ip') - instance_id = server.get('instance_id', 'Null') - hostname = server.get('hostname', instance_id) - if not hostname.strip(): - hostname = instance_id - if re.search('syncserver', hostname): - hostname = '{}_{}'.format(hostname, private_ip) - - region = server.get('region', 'Null') - instance_type = server.get('instance_type', 'Null') - instance_state = server.get('instance_state', 'Null') - cpu = server.get('cpu', 'Null') - cpu_cores = server.get('cpu_cores', 'Null') - memory = server.get('memory', 'Null') - disk = server.get('disk', 'Null') - os_type = server.get('os_type', 'Null') - os_kernel = server.get('os_kernel', 'Null') - sn = server.get('sn', 'Null') - - exist_hostname = session.query(Server).filter(Server.hostname == hostname).first() - exist_ip = session.query(Server).filter(Server.ip == ip).first() - - if not exist_hostname and not exist_ip: - new_serve = Server(ip=ip, public_ip=ip, private_ip=private_ip, hostname=hostname, port=22, - idc=self.account, region=region, - state=self.state, - admin_user=self.default_admin_user) - new_serve_detail = ServerDetail(ip=ip, instance_id=instance_id, instance_type=instance_type, - instance_state=instance_state, cpu=cpu, cpu_cores=cpu_cores, - memory=memory, - disk=disk, os_type=os_type, os_kernel=os_kernel, sn=sn) - session.add(new_serve) - session.add(new_serve_detail) - else: - session.query(Server).filter(Server.hostname == hostname).update( - {Server.ip: ip, Server.private_ip: private_ip, Server.public_ip: ip, Server.idc: self.account, - Server.region: region, - Server.admin_user: self.default_admin_user}) - session.query(ServerDetail).filter(ServerDetail.ip == ip).update( - {ServerDetail.instance_id: instance_id, ServerDetail.instance_type: instance_type, - ServerDetail.instance_state: instance_state}) - - session.commit() - - def test_auth(self): - """ - 测试下用户给的信息是否正确 - :return: - """ - self.offset = '0' - self.limit = '1' # 测试的时候只请求一个机器就可以了，不然机器多了就卡好长时间 - result_url = self.get_server_url() - response = requests.get(result_url) - result_data = json.loads(response.text) - return result_data - - def index(self): - """ - 腾讯云若机器超过100台需要进行通过offset+limit获取 - :return: - """ - count = self.get_server_count() - # print('Tocal：{}'.format(count)) - for c in range(0, count, 100): - self.offset = str(c) - if (c + 100) > count: - self.limit = str(count) - else: - self.limit = str(c + 100) - ins_log.read_log('info', '开始同步腾讯云的第{}--{}台机器'.format(self.offset, self.limit)) - self.sync_cmdb() - - -def get_configs(): - """ - get id / key / region info - :return: - """ - - qcloud_configs_list = [] - with DBContext('r') as session: - qcloud_configs_info = session.query(AssetConfigs).filter(AssetConfigs.account == '腾讯云', - AssetConfigs.state == 'true').all() - for data in qcloud_configs_info: - data_dict = model_to_dict(data) - data_dict['create_time'] = str(data_dict['create_time']) - data_dict['update_time'] = str(data_dict['update_time']) - qcloud_configs_list.append(data_dict) - return qcloud_configs_list - - -def main(): - """ - 从接口获取已经启用的配置 - :return: - """ - - mc = MyCryptV2() - qcloud_configs_list = get_configs() - if not qcloud_configs_list: - ins_log.read_log('error', '没有获取到腾讯云资产配置信息，跳过') - return False - for config in qcloud_configs_list: - access_id = config.get('access_id') - access_key = mc.my_decrypt(config.get('access_key')) # 解密后使用 - region = config.get('region') - default_admin_user = config.get('default_admin_user') - - obj = CVMApi(access_id, access_key, region, default_admin_user) - obj.index() - - -if __name__ == '__main__': - fire.Fire(main) +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Time : 2019/5/10 16:37 +# @Author : Fred Yangxiaofei +# @File : cvm.py +# @Role : 获取腾讯云主机信息 + + +import time +import random +import requests +import json +import re +from libs.qcloud.qcloud_api import ApiOper +from libs.db_context import DBContext +from libs.web_logs import ins_log +from opssdk.operate import MyCryptV2 +from models.server import Server, ServerDetail, AssetConfigs, model_to_dict +import fire + + +class CVMApi(): + def __init__(self, access_id, access_key, region, default_admin_user): + self.offset = '0' # 偏移量,这里拼接的时候必须是字符串 + self.limit = '100' # 官方默认是20，大于100需要设置偏移量再次请求：offset=100,offset={机器总数} + self.access_id = access_id + self.access_key = access_key + self.region = region + self.default_admin_user = default_admin_user # 默认管理用户 + self.account = '腾讯云' # 标记机器为腾讯云 + self.state = 'auto' # 标记自动获取状态 + + def get_server_url(self): + """ + 获取腾讯云机器的API拼接结果，get请求 + :return: + """ + api_url = 'cvm.tencentcloudapi.com/?' + keydict = { + # 公共参数部分 + 'Timestamp': str(int(time.time())), + 'Nonce': str(int(random.random() * 1000)), + 'Region': self.region, + 'SecretId': self.access_id, + 'Version': '2017-03-12', + # 'SignatureMethod': SignatureMethod, + # 接口参数部分 + 'Action': 'DescribeInstances', + 'Offset': self.offset, # 机器数量超过100需要用到偏移量 + 'Limit': self.limit, + } + result_url = ApiOper.run(keydict, api_url, self.access_key) + return result_url + + def get_result_data(self): + """ + 获取返回的数据 + :return: + """ + result_url = self.get_server_url() + response = requests.get(result_url) + result_data = json.loads(response.text) + if result_data['Response'].get('Error'): + ins_log.read_log('error', '{}'.format(result_data['Response'])) + return False + else: + ret = result_data['Response'] + + return ret + + def get_server_count(self): + """ + 获取主机总数量。如果机器数量大于limit限制就要用到offset偏移查询 + :return: int + """ + + ret = self.get_result_data() + if not ret: + return False + + server_count = ret['TotalCount'] + return server_count + + def get_server_info(self): + """ + 获取QCloud服务器信息 + :return: + """ + + ret = self.get_result_data() + if not ret: + return False + + server_list = [] + + server_info = ret['InstanceSet'] + for i in server_info: + asset_data = dict() + instance_id = i.get('InstanceId') + instance_state = i.get('InstanceState') + instance_type = i.get('InstanceType') + cpu = i.get('CPU') + memory = i.get('Memory') + hostname = i.get('InstanceName') + disk = i['SystemDisk'].get('DiskSize') + try: + private_ip = i['PrivateIpAddresses'][0] + except (KeyError, TypeError): + private_ip = 'Null' + try: + public_ip = i['PublicIpAddresses'][0] + except (KeyError, TypeError): + public_ip = private_ip # 不存在公网就给私网IP + os_type = i.get('OsName') + region = i['Placement'].get('Zone') + asset_data['region'] = region + asset_data['instance_id'] = instance_id + asset_data['instance_state'] = instance_state + asset_data['instance_type'] = instance_type + asset_data['cpu_cores'] = '{}Core'.format(cpu) + asset_data['memory'] = '{}G'.format(memory) + asset_data['hostname'] = hostname + asset_data['disk'] = '{}G'.format(disk) + asset_data['private_ip'] = private_ip + asset_data['public_ip'] = public_ip + asset_data['os_type'] = os_type + # print(asset_data) + server_list.append(asset_data) + ins_log.read_log('info', '资产信息:{}'.format(asset_data)) + return server_list + + def sync_cmdb(self): + """ + 返回结果写入CMDB + :return: + """ + server_list = self.get_server_info() + + if not server_list: + print('Not Fount Server Info') + return False + + with DBContext('w') as session: + for server in server_list: + ip = server.get('public_ip') + private_ip = server.get('private_ip') + instance_id = server.get('instance_id', 'Null') + hostname = server.get('hostname', instance_id) + if not hostname.strip(): + hostname = instance_id + if re.search('syncserver', hostname): + hostname = '{}_{}'.format(hostname, private_ip) + + region = server.get('region', 'Null') + instance_type = server.get('instance_type', 'Null') + instance_state = server.get('instance_state', 'Null') + cpu = server.get('cpu', 'Null') + cpu_cores = server.get('cpu_cores', 'Null') + memory = server.get('memory', 'Null') + disk = server.get('disk', 'Null') + os_type = server.get('os_type', 'Null') + os_kernel = server.get('os_kernel', 'Null') + sn = server.get('sn', 'Null') + + exist_hostname = session.query(Server).filter(Server.hostname == hostname).first() + exist_ip = session.query(Server).filter(Server.ip == ip).first() + + if not exist_hostname and not exist_ip: + new_serve = Server(ip=ip, public_ip=ip, private_ip=private_ip, hostname=hostname, port=22, + idc=self.account, region=region, + state=self.state, + admin_user=self.default_admin_user) + new_serve_detail = ServerDetail(ip=ip, instance_id=instance_id, instance_type=instance_type, + instance_state=instance_state, cpu=cpu, cpu_cores=cpu_cores, + memory=memory, + disk=disk, os_type=os_type, os_kernel=os_kernel, sn=sn) + session.add(new_serve) + session.add(new_serve_detail) + else: + session.query(Server).filter(Server.hostname == hostname).update( + {Server.ip: ip, Server.private_ip: private_ip, Server.public_ip: ip, Server.idc: self.account, + Server.region: region, + Server.admin_user: self.default_admin_user}) + session.query(ServerDetail).filter(ServerDetail.ip == ip).update( + {ServerDetail.instance_id: instance_id, ServerDetail.instance_type: instance_type, + ServerDetail.instance_state: instance_state}) + + session.commit() + + def test_auth(self): + """ + 测试下用户给的信息是否正确 + :return: + """ + self.offset = '0' + self.limit = '1' # 测试的时候只请求一个机器就可以了，不然机器多了就卡好长时间 + result_url = self.get_server_url() + response = requests.get(result_url) + result_data = json.loads(response.text) + return result_data + + def index(self): + """ + 腾讯云若机器超过100台需要进行通过offset+limit获取 + :return: + """ + count = self.get_server_count() + # print('Tocal：{}'.format(count)) + for c in range(0, count, 100): + self.offset = str(c) + if (c + 100) > count: + self.limit = str(count) + else: + self.limit = str(c + 100) + ins_log.read_log('info', '开始同步腾讯云的第{}--{}台机器'.format(self.offset, self.limit)) + self.sync_cmdb() + + +def get_configs(): + """ + get id / key / region info + :return: + """ + + qcloud_configs_list = [] + with DBContext('r') as session: + qcloud_configs_info = session.query(AssetConfigs).filter(AssetConfigs.account == '腾讯云', + AssetConfigs.state == 'true').all() + for data in qcloud_configs_info: + data_dict = model_to_dict(data) + data_dict['create_time'] = str(data_dict['create_time']) + data_dict['update_time'] = str(data_dict['update_time']) + qcloud_configs_list.append(data_dict) + return qcloud_configs_list + + +def main(): + """ + 从接口获取已经启用的配置 + :return: + """ + + mc = MyCryptV2() + qcloud_configs_list = get_configs() + if not qcloud_configs_list: + ins_log.read_log('error', '没有获取到腾讯云资产配置信息，跳过') + return False + for config in qcloud_configs_list: + access_id = config.get('access_id') + access_key = mc.my_decrypt(config.get('access_key')) # 解密后使用 + region = config.get('region') + default_admin_user = config.get('default_admin_user') + + obj = CVMApi(access_id, access_key, region, default_admin_user) + obj.index() + + +if __name__ == '__main__': + fire.Fire(main) diff --git a/libs/qcloud/qcloud_api.py b/libs/qcloud/qcloud_api.py index 4c2a3c5..70a92fb 100644 --- a/libs/qcloud/qcloud_api.py +++ b/libs/qcloud/qcloud_api.py @@ -1,105 +1,105 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# @Time : 7/17/2018 9:41 AM -# @Author : Fred Yang -# @File : qcloud_api.py -# @Role : 腾讯云API操作接口 - -import hmac -import hashlib -import base64 -from urllib import parse - -class ApiOper(object): - """ - 说明： - 腾讯云API的签名要进行：构造参数字典->对dict排序->拼接sign->对sign编码->拼接完成最终url->完成调用 - - 使用： - 构建参数字典部分通过其余脚本传参进来（如下例子） - def get_dict(): - keydict = { - 'Action': 'GetCdbExportLogUrl', - 'Timestamp': str(int(time.time())), - 'Nonce': str(int(random.random() * 1000)), - 'Region': 'ap-shanghai', - 'SecretId': TX_INFO['SecretId'], - # 'SignatureMethod': SignatureMethod, - 'cdbInstanceId': 'cdb-extlv472', - 'type': 'coldbackup' - } - return keydict - """ - - @staticmethod - def sort_dic(keydict): - """ - 对字典进行拼接 - :param keydict: - :return: 返回排序后的列表 - """ - sortlist = sorted(zip(keydict.keys(), keydict.values())) - return sortlist - - @staticmethod - def get_str_sign(sortlist, api_url): - """ - 将排序后的列表进行字符串拼接 - :param sortlist: - :return: 拼接后的字符串 - """ - sign_str_init = '' - for value in sortlist: - sign_str_init += value[0] + '=' + value[1] + '&' - sign_str = 'GET' + api_url + sign_str_init[:-1] - return sign_str, sign_str_init - - @staticmethod - def get_signature(sign_str,secret_key): - """ - 生成签名 - :param sign_str: - :param secretkey: - :return:签名字符串 - """ - secretkey = secret_key - signature = bytes(sign_str, encoding='utf-8') - secretkey = bytes(secretkey, encoding='utf-8') - my_sign = hmac.new(secretkey, signature, hashlib.sha1).digest() - my_sign = base64.b64encode(my_sign) - return my_sign - - @staticmethod - def encode_signature( my_sign): - """ - 对签名编码 - :param my_sign: - :return: 编码后的签名串 - """ - result_sign = parse.quote(my_sign) - return result_sign - - @staticmethod - def get_result_url(sign_str, result_sign,api_url): - """ - 完成最终url拼接 - :param result_sign: - :return: 最终url - """ - result_url = 'https://' + api_url + sign_str + 'Signature=' + result_sign - return result_url - - @staticmethod - def run(keydict, api_url,secret_key): - # 获取请求参数dict(使用脚本传进来) - # 对参数dict进行排序 - sortlist = ApiOper.sort_dic(keydict) - # 获取拼接后的sign字符串 - sign_str, sign_str_int = ApiOper.get_str_sign(sortlist, api_url) - # 获取签名 - my_sign = ApiOper.get_signature(sign_str,secret_key) - # 对签名串进行编码 - result_sign = ApiOper.encode_signature(my_sign) - # 获取最终请求url - result_url = ApiOper.get_result_url(sign_str_int, result_sign,api_url) - return result_url +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Time : 7/17/2018 9:41 AM +# @Author : Fred Yang +# @File : qcloud_api.py +# @Role : 腾讯云API操作接口 + +import hmac +import hashlib +import base64 +from urllib import parse + +class ApiOper(object): + """ + 说明： + 腾讯云API的签名要进行：构造参数字典->对dict排序->拼接sign->对sign编码->拼接完成最终url->完成调用 + + 使用： + 构建参数字典部分通过其余脚本传参进来（如下例子） + def get_dict(): + keydict = { + 'Action': 'GetCdbExportLogUrl', + 'Timestamp': str(int(time.time())), + 'Nonce': str(int(random.random() * 1000)), + 'Region': 'ap-shanghai', + 'SecretId': TX_INFO['SecretId'], + # 'SignatureMethod': SignatureMethod, + 'cdbInstanceId': 'cdb-extlv472', + 'type': 'coldbackup' + } + return keydict + """ + + @staticmethod + def sort_dic(keydict): + """ + 对字典进行拼接 + :param keydict: + :return: 返回排序后的列表 + """ + sortlist = sorted(zip(keydict.keys(), keydict.values())) + return sortlist + + @staticmethod + def get_str_sign(sortlist, api_url): + """ + 将排序后的列表进行字符串拼接 + :param sortlist: + :return: 拼接后的字符串 + """ + sign_str_init = '' + for value in sortlist: + sign_str_init += value[0] + '=' + value[1] + '&' + sign_str = 'GET' + api_url + sign_str_init[:-1] + return sign_str, sign_str_init + + @staticmethod + def get_signature(sign_str,secret_key): + """ + 生成签名 + :param sign_str: + :param secretkey: + :return:签名字符串 + """ + secretkey = secret_key + signature = bytes(sign_str, encoding='utf-8') + secretkey = bytes(secretkey, encoding='utf-8') + my_sign = hmac.new(secretkey, signature, hashlib.sha1).digest() + my_sign = base64.b64encode(my_sign) + return my_sign + + @staticmethod + def encode_signature( my_sign): + """ + 对签名编码 + :param my_sign: + :return: 编码后的签名串 + """ + result_sign = parse.quote(my_sign) + return result_sign + + @staticmethod + def get_result_url(sign_str, result_sign,api_url): + """ + 完成最终url拼接 + :param result_sign: + :return: 最终url + """ + result_url = 'https://' + api_url + sign_str + 'Signature=' + result_sign + return result_url + + @staticmethod + def run(keydict, api_url,secret_key): + # 获取请求参数dict(使用脚本传进来) + # 对参数dict进行排序 + sortlist = ApiOper.sort_dic(keydict) + # 获取拼接后的sign字符串 + sign_str, sign_str_int = ApiOper.get_str_sign(sortlist, api_url) + # 获取签名 + my_sign = ApiOper.get_signature(sign_str,secret_key) + # 对签名串进行编码 + result_sign = ApiOper.encode_signature(my_sign) + # 获取最终请求url + result_url = ApiOper.get_result_url(sign_str_int, result_sign,api_url) + return result_url diff --git a/libs/script/__init__.py b/libs/script/__init__.py index 84540fa..5f69d24 100644 --- a/libs/script/__init__.py +++ b/libs/script/__init__.py @@ -1,6 +1,6 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# @Time : 2019/4/24 10:50 -# @Author : Fred Yangxiaofei -# @File : __init__.py.py +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Time : 2019/4/24 10:50 +# @Author : Fred Yangxiaofei +# @File : __init__.py.py # @Role : 说明脚本功能 \ No newline at end of file diff --git a/libs/script/multi_hosts.py b/libs/script/multi_hosts.py index 7437c3e..b459b6c 100644 --- a/libs/script/multi_hosts.py +++ b/libs/script/multi_hosts.py @@ -1,42 +1,42 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# @Time : 2019/7/12 9:39 -# @Author : Fred Yangxiaofei -# @File : multi_hosts.py -# @Role : API批量添加主机 - -import json -import requests - - -def add_host(): - add_user_url = 'https://codo.domain.com/api/cmdb2/v1/cmdb/server/' - csrf_task_url = 'https://codo.domain.com/api/task/v2/task/accept/' - - # 这里就是一个长期Token，管理员可以在用户列表选择一个用户进行生成一个长期Token - auth_key = "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9nVzZXJuYW1lIjoieWFuZ2gFPLN6g" - - the_body = json.dumps({ - "hostname": 'hostname', - "ip": "1.1.1.1", - "port": "22", - "public_ip": "2.2.2.2", - "idc": "AWS", - "admin_user": "root", - "region": "us-east-1", - }) - - req1 = requests.get(csrf_task_url, cookies=dict(auth_key=auth_key)) - csrf_key = json.loads(req1.text)['csrf_key'] - cookies = dict(auth_key=auth_key, csrf_key=csrf_key) - req = requests.post(add_user_url, data=the_body, cookies=cookies) - req_code = json.loads(req.text)['code'] - if req_code != 0: - print(json.loads(req.text)['msg']) - exit(-111) - else: - print(json.loads(req.text)['msg']) - - -if __name__ == '__main__': - add_host() +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Time : 2019/7/12 9:39 +# @Author : Fred Yangxiaofei +# @File : multi_hosts.py +# @Role : API批量添加主机 + +import json +import requests + + +def add_host(): + add_user_url = 'https://codo.domain.com/api/cmdb2/v1/cmdb/server/' + csrf_task_url = 'https://codo.domain.com/api/task/v2/task/accept/' + + # 这里就是一个长期Token，管理员可以在用户列表选择一个用户进行生成一个长期Token + auth_key = "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9nVzZXJuYW1lIjoieWFuZ2gFPLN6g" + + the_body = json.dumps({ + "hostname": 'hostname', + "ip": "1.1.1.1", + "port": "22", + "public_ip": "2.2.2.2", + "idc": "AWS", + "admin_user": "root", + "region": "us-east-1", + }) + + req1 = requests.get(csrf_task_url, cookies=dict(auth_key=auth_key)) + csrf_key = json.loads(req1.text)['csrf_key'] + cookies = dict(auth_key=auth_key, csrf_key=csrf_key) + req = requests.post(add_user_url, data=the_body, cookies=cookies) + req_code = json.loads(req.text)['code'] + if req_code != 0: + print(json.loads(req.text)['msg']) + exit(-111) + else: + print(json.loads(req.text)['msg']) + + +if __name__ == '__main__': + add_host() diff --git a/libs/script/sysinfo.py b/libs/script/sysinfo.py index 37cbb0e..7b91351 100644 --- a/libs/script/sysinfo.py +++ b/libs/script/sysinfo.py @@ -1,81 +1,81 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# @Time : 2019/4/24 10:50 -# @Author : Fred Yangxiaofei -# @File : sysinfo.py -# @Role : 收集Linux基础信息 -# @备注:不用Python收集是因为不想给客户机器装模块，影响用户机器环境等，直接用Linux Bash取值 -# 二更：现在用的是Ansible获取，此脚本可用暂且不用 -import subprocess -import json - - -def exec_shell(cmd): - '''执行shell命令函数''' - sub2 = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - stdout, stderr = sub2.communicate() - ret = sub2.returncode - if ret == 0: - return stdout.decode('utf-8').strip() - else: - return None - - -def yum_check(): - yum_list = ['redhat-lsb', 'dmidecode'] - for i in yum_list: - cmd = "rpm -qa | grep %s ; [ $? -ne '0' ] && yum install -y %s && echo '%s install ok'" % (i, i, i) - exec_shell(cmd) - - -def collect(): - """ - 收集资产基础信息 - :return: - """ - data = {} - data['sn'] = get_sn() # SN - data['hostname'] = exec_shell("hostname") # 主机名 - data['os_distribution'] = get_os_distributor() # OS版本，如:Centos Ubuntu - data['os_version'] = get_os_version() # 系统版本 - data['cpu'] = exec_shell("cat /proc/cpuinfo | grep 'cpu cores' | uniq | awk {'print $4'}") + 'Core' # CPU核心数 - data['memory'] = get_memory() - data['disk'] = exec_shell("df --total -h | grep total | awk {'print $2'}") # 总空间 - data['disk_utilization'] = exec_shell("df --total -h | grep total | awk {'print $5'}") # 总使用百分比 - return json.dumps(data) - - -def get_sn(): - '''获取SN''' - cmd_res = exec_shell("dmidecode -t system|grep 'Serial Number'") - cmd_res = cmd_res.strip() - res_to_list = cmd_res.split(':') - if len(res_to_list) > 1: # the second one is wanted string - return res_to_list[1].strip() - else: - return '' - -# def get_cpu(): -# '''获取CPU信息''' -# status, output = exec_shell() - -def get_memory(): - '''获取内存信息''' - distributor = exec_shell("free | grep Mem | grep -v total | awk {'print $2'}") - return (int(distributor) / 1024 / 1024) - -def get_os_distributor(): - '''获取OS类型''' - distributor = exec_shell("lsb_release -a|grep 'Distributor ID'").split(":") - return distributor[1].strip() if len(distributor) > 1 else None - - -def get_os_version(): - '''获取系统版本''' - version = exec_shell("lsb_release -a | grep 'Release'").split(":") - return version[1].strip() if len(version) > 1 else None - - -if __name__ == "__main__": - yum_check() - print(collect()) +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Time : 2019/4/24 10:50 +# @Author : Fred Yangxiaofei +# @File : sysinfo.py +# @Role : 收集Linux基础信息 +# @备注:不用Python收集是因为不想给客户机器装模块，影响用户机器环境等，直接用Linux Bash取值 +# 二更：现在用的是Ansible获取，此脚本可用暂且不用 +import subprocess +import json + + +def exec_shell(cmd): + '''执行shell命令函数''' + sub2 = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + stdout, stderr = sub2.communicate() + ret = sub2.returncode + if ret == 0: + return stdout.decode('utf-8').strip() + else: + return None + + +def yum_check(): + yum_list = ['redhat-lsb', 'dmidecode'] + for i in yum_list: + cmd = "rpm -qa | grep %s ; [ $? -ne '0' ] && yum install -y %s && echo '%s install ok'" % (i, i, i) + exec_shell(cmd) + + +def collect(): + """ + 收集资产基础信息 + :return: + """ + data = {} + data['sn'] = get_sn() # SN + data['hostname'] = exec_shell("hostname") # 主机名 + data['os_distribution'] = get_os_distributor() # OS版本，如:Centos Ubuntu + data['os_version'] = get_os_version() # 系统版本 + data['cpu'] = exec_shell("cat /proc/cpuinfo | grep 'cpu cores' | uniq | awk {'print $4'}") + 'Core' # CPU核心数 + data['memory'] = get_memory() + data['disk'] = exec_shell("df --total -h | grep total | awk {'print $2'}") # 总空间 + data['disk_utilization'] = exec_shell("df --total -h | grep total | awk {'print $5'}") # 总使用百分比 + return json.dumps(data) + + +def get_sn(): + '''获取SN''' + cmd_res = exec_shell("dmidecode -t system|grep 'Serial Number'") + cmd_res = cmd_res.strip() + res_to_list = cmd_res.split(':') + if len(res_to_list) > 1: # the second one is wanted string + return res_to_list[1].strip() + else: + return '' + +# def get_cpu(): +# '''获取CPU信息''' +# status, output = exec_shell() + +def get_memory(): + '''获取内存信息''' + distributor = exec_shell("free | grep Mem | grep -v total | awk {'print $2'}") + return (int(distributor) / 1024 / 1024) + +def get_os_distributor(): + '''获取OS类型''' + distributor = exec_shell("lsb_release -a|grep 'Distributor ID'").split(":") + return distributor[1].strip() if len(distributor) > 1 else None + + +def get_os_version(): + '''获取系统版本''' + version = exec_shell("lsb_release -a | grep 'Release'").split(":") + return version[1].strip() if len(version) > 1 else None + + +if __name__ == "__main__": + yum_check() + print(collect()) diff --git a/libs/server/__init__.py b/libs/server/__init__.py index 2554ed1..e001512 100644 --- a/libs/server/__init__.py +++ b/libs/server/__init__.py @@ -1,6 +1,6 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# @Time : 2019/5/1 13:28 -# @Author : Fred Yangxiaofei -# @File : __init__.py.py -# @Role : 通过Ansible API推送密钥，获取主机信息 +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Time : 2019/5/1 13:28 +# @Author : Fred Yangxiaofei +# @File : __init__.py.py +# @Role : 通过Ansible API推送密钥，获取主机信息 diff --git a/libs/server/asset_auto_update.py b/libs/server/asset_auto_update.py index b08c83b..3edd762 100644 --- a/libs/server/asset_auto_update.py +++ b/libs/server/asset_auto_update.py @@ -1,206 +1,206 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# @Time : 2019/4/26 14:34 -# @Author : Fred Yangxiaofei -# @File : test_asset_update.py -# @Role : 自动获取资产信息 - - -from libs.db_context import DBContext -from models.server import Server, ServerDetail, model_to_dict, AdminUser, AssetErrorLog -from libs.server.sync_public_key import RsyncPublicKey, start_rsync -from libs.server.collect_asset_info import get_server_sysinfo -from libs.web_logs import ins_log -import sqlalchemy -import fire - - -class AssetServerAUtoUpdate(): - def __init__(self, state): - self.state = state - - def check_server_state(self): - '''查询Server状态, 新建的推送Key并更新''' - new_asset_list = [] - id_list = [] - with DBContext('r') as session: - # 新加的资产,state:new - new_asset = session.query(Server).filter(Server.state == self.state).all() - for msg in new_asset: - data_dict = model_to_dict(msg) - data_dict['create_time'] = str(data_dict['create_time']) - new_asset_list.append(data_dict) - - for i in new_asset_list: - id_list.append(i.get('id')) - - return id_list - - def rsync_public_key(self): - '''推送公钥到新加的主机里面''' - id_list = self.check_server_state() - if not id_list: - ins_log.read_log('info', '[PASS]: No new server found, automatically skipping push public key') - # print('[PASS]: No new server found, automatically skipping push public key') - - return - - # 根据ID列表查询,获取管理权限推送PublicKey到主机 - new_server_list = [] - rsync_sucess_list = [] - rsync_error_list = [] - with DBContext('r') as session: - for i in id_list: - connect_info = session.query(Server.ip, Server.port, AdminUser.system_user, - AdminUser.user_key).outerjoin(AdminUser, - AdminUser.admin_user == Server.admin_user).filter( - Server.id == i).all() - new_server_list.append(connect_info) - # 检查密钥 - sync_key_obj = RsyncPublicKey() - check = sync_key_obj.check_rsa() - if check: - # print('new_server_list-->', new_server_list) - res_data = start_rsync(new_server_list) - # print(res_data) - for res in res_data: - if not res.get('status'): - rsync_error_list.append(res) - else: - # 只返回密钥推送成功的进行更新资产 - rsync_sucess_list.append(res) - - if rsync_error_list: - with DBContext('w') as session: - for i in rsync_error_list: - ip = i.get('ip') - msg = i.get('msg') - error_log = '推送公钥失败, 错误信息：{}'.format(msg) - ins_log.read_log('error', error_log) - session.query(Server).filter(Server.ip == ip).update({Server.state: 'false'}) - exist_ip = session.query(AssetErrorLog).filter(AssetErrorLog.ip == ip).first() - if exist_ip: - session.query(AssetErrorLog).filter(AssetErrorLog.ip == ip).update( - {AssetErrorLog.error_log: error_log}) - else: - new_error_log = AssetErrorLog(ip=ip, error_log=error_log) - session.add(new_error_log) - session.commit() - - return rsync_sucess_list - - def update_asset(self, host_info): - """ - 更新资产到数据库 - :param host_data: 主机返回的资产采集基础数据 - :return: - """ - with DBContext('w') as session: - for k, v in host_info.items(): - try: - if host_info[k].get('status'): - _sn = v.get('sn', None) - _hostname = v.get('host_name', None) - _cpu = v.get('cpu', None) - _cpu_cores = v.get('cpu_cores', None) - _memory = v.get('memory', None) - _disk = v.get('disk', None) - _os_type = v.get('os_type', None) - _os_kernel = v.get('os_kernel', None) - # _instance_id = v.get('instance_id', None) - # _instance_type = v.get('instance_type', None) - # _instance_state = v.get('instance_state', None) - - exist_detail = session.query(ServerDetail).filter(ServerDetail.ip == k).first() - if not exist_detail: - # 不存在就新建 - new_server_detail = ServerDetail(ip=k, sn=_sn, cpu=_cpu, cpu_cores=_cpu_cores, - memory=_memory, disk=_disk, - os_type=_os_type, os_kernel=_os_kernel) - session.add(new_server_detail) - session.commit() - session.query(Server).filter(Server.ip == k).update( - {Server.hostname: _hostname, Server.state: 'true'}) - session.commit() - else: - # 存在就更新 - session.query(ServerDetail).filter(ServerDetail.ip == k).update({ - ServerDetail.sn: _sn, ServerDetail.ip: k, - ServerDetail.cpu: _cpu, ServerDetail.cpu_cores: _cpu_cores, - ServerDetail.disk: _disk, ServerDetail.memory: _memory, - ServerDetail.os_type: _os_type, ServerDetail.os_kernel: _os_kernel, - }) - - session.query(Server).filter(Server.ip == k).update( - {Server.hostname: _hostname, Server.state: 'true'}) - session.commit() - except sqlalchemy.exc.IntegrityError as e: - ins_log.read_log('error', e) - # 状态改为Flse->删除主机Detail--记录错误信息 - session.query(Server).filter(Server.ip == k).update({Server.state: 'false'}) - session.query(ServerDetail).filter(ServerDetail.ip == k).delete( - synchronize_session=False) - - exist_ip = session.query(AssetErrorLog).filter(AssetErrorLog.ip == k).first() - error_log = str(e) - if exist_ip: - session.query(AssetErrorLog).filter(AssetErrorLog.ip == k).update( - {AssetErrorLog.error_log: error_log}) - else: - new_error_log = AssetErrorLog(ip=k, error_log=error_log) - session.add(new_error_log) - - session.commit() - return False - - def get_host_info(self): - '''获取主机信息，并写入数据库''' - id_list = self.check_server_state() - with DBContext('r') as session: - for i in id_list: - server_list = session.query(Server.ip, Server.port, AdminUser.system_user).outerjoin(AdminUser, - AdminUser.admin_user == Server.admin_user).filter( - Server.id == i).all() - asset_data = get_server_sysinfo(server_list) - ins_log.read_log('info', '资产信息：{}'.format(asset_data)) - self.update_asset(asset_data) - - -def main(state): - """ - 机器状态,分为:new, true, false - new: 表示新加的机器 - 1. 先推送密钥 - 2. 多进程使用absible获取主机资产信息 - 3. 将资产信息写入/更新到数据库 - - true: 表示已经可以正常免密钥登陆 - 1. 多进程使用absible获取主机资产信息 - 2. 将资产信息写入/更新到数据库 - - false: 表示主机可能配置有问题,无法正常更新资产 - 1. 将错误信息记录数据库 - :param state: 机器的状态 - :return: - """ - - obj = AssetServerAUtoUpdate(state) - if state == 'new': - if not obj.rsync_public_key(): - # 如果没有发现有新增的主机,直接PASS - return - obj.get_host_info() - elif state == 'true': - obj.get_host_info() - - -def new_tail_data(): - main('new') - - -def true_tail_data(): - main('true') - - -if __name__ == '__main__': - fire.Fire(main) +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Time : 2019/4/26 14:34 +# @Author : Fred Yangxiaofei +# @File : test_asset_update.py +# @Role : 自动获取资产信息 + + +from libs.db_context import DBContext +from models.server import Server, ServerDetail, model_to_dict, AdminUser, AssetErrorLog +from libs.server.sync_public_key import RsyncPublicKey, start_rsync +from libs.server.collect_asset_info import get_server_sysinfo +from libs.web_logs import ins_log +import sqlalchemy +import fire + + +class AssetServerAUtoUpdate(): + def __init__(self, state): + self.state = state + + def check_server_state(self): + '''查询Server状态, 新建的推送Key并更新''' + new_asset_list = [] + id_list = [] + with DBContext('r') as session: + # 新加的资产,state:new + new_asset = session.query(Server).filter(Server.state == self.state).all() + for msg in new_asset: + data_dict = model_to_dict(msg) + data_dict['create_time'] = str(data_dict['create_time']) + new_asset_list.append(data_dict) + + for i in new_asset_list: + id_list.append(i.get('id')) + + return id_list + + def rsync_public_key(self): + '''推送公钥到新加的主机里面''' + id_list = self.check_server_state() + if not id_list: + ins_log.read_log('info', '[PASS]: No new server found, automatically skipping push public key') + # print('[PASS]: No new server found, automatically skipping push public key') + + return + + # 根据ID列表查询,获取管理权限推送PublicKey到主机 + new_server_list = [] + rsync_sucess_list = [] + rsync_error_list = [] + with DBContext('r') as session: + for i in id_list: + connect_info = session.query(Server.ip, Server.port, AdminUser.system_user, + AdminUser.user_key).outerjoin(AdminUser, + AdminUser.admin_user == Server.admin_user).filter( + Server.id == i).all() + new_server_list.append(connect_info) + # 检查密钥 + sync_key_obj = RsyncPublicKey() + check = sync_key_obj.check_rsa() + if check: + # print('new_server_list-->', new_server_list) + res_data = start_rsync(new_server_list) + # print(res_data) + for res in res_data: + if not res.get('status'): + rsync_error_list.append(res) + else: + # 只返回密钥推送成功的进行更新资产 + rsync_sucess_list.append(res) + + if rsync_error_list: + with DBContext('w') as session: + for i in rsync_error_list: + ip = i.get('ip') + msg = i.get('msg') + error_log = '推送公钥失败, 错误信息：{}'.format(msg) + ins_log.read_log('error', error_log) + session.query(Server).filter(Server.ip == ip).update({Server.state: 'false'}) + exist_ip = session.query(AssetErrorLog).filter(AssetErrorLog.ip == ip).first() + if exist_ip: + session.query(AssetErrorLog).filter(AssetErrorLog.ip == ip).update( + {AssetErrorLog.error_log: error_log}) + else: + new_error_log = AssetErrorLog(ip=ip, error_log=error_log) + session.add(new_error_log) + session.commit() + + return rsync_sucess_list + + def update_asset(self, host_info): + """ + 更新资产到数据库 + :param host_data: 主机返回的资产采集基础数据 + :return: + """ + with DBContext('w') as session: + for k, v in host_info.items(): + try: + if host_info[k].get('status'): + _sn = v.get('sn', None) + _hostname = v.get('host_name', None) + _cpu = v.get('cpu', None) + _cpu_cores = v.get('cpu_cores', None) + _memory = v.get('memory', None) + _disk = v.get('disk', None) + _os_type = v.get('os_type', None) + _os_kernel = v.get('os_kernel', None) + # _instance_id = v.get('instance_id', None) + # _instance_type = v.get('instance_type', None) + # _instance_state = v.get('instance_state', None) + + exist_detail = session.query(ServerDetail).filter(ServerDetail.ip == k).first() + if not exist_detail: + # 不存在就新建 + new_server_detail = ServerDetail(ip=k, sn=_sn, cpu=_cpu, cpu_cores=_cpu_cores, + memory=_memory, disk=_disk, + os_type=_os_type, os_kernel=_os_kernel) + session.add(new_server_detail) + session.commit() + session.query(Server).filter(Server.ip == k).update( + {Server.hostname: _hostname, Server.state: 'true'}) + session.commit() + else: + # 存在就更新 + session.query(ServerDetail).filter(ServerDetail.ip == k).update({ + ServerDetail.sn: _sn, ServerDetail.ip: k, + ServerDetail.cpu: _cpu, ServerDetail.cpu_cores: _cpu_cores, + ServerDetail.disk: _disk, ServerDetail.memory: _memory, + ServerDetail.os_type: _os_type, ServerDetail.os_kernel: _os_kernel, + }) + + session.query(Server).filter(Server.ip == k).update( + {Server.hostname: _hostname, Server.state: 'true'}) + session.commit() + except sqlalchemy.exc.IntegrityError as e: + ins_log.read_log('error', e) + # 状态改为Flse->删除主机Detail--记录错误信息 + session.query(Server).filter(Server.ip == k).update({Server.state: 'false'}) + session.query(ServerDetail).filter(ServerDetail.ip == k).delete( + synchronize_session=False) + + exist_ip = session.query(AssetErrorLog).filter(AssetErrorLog.ip == k).first() + error_log = str(e) + if exist_ip: + session.query(AssetErrorLog).filter(AssetErrorLog.ip == k).update( + {AssetErrorLog.error_log: error_log}) + else: + new_error_log = AssetErrorLog(ip=k, error_log=error_log) + session.add(new_error_log) + + session.commit() + return False + + def get_host_info(self): + '''获取主机信息，并写入数据库''' + id_list = self.check_server_state() + with DBContext('r') as session: + for i in id_list: + server_list = session.query(Server.ip, Server.port, AdminUser.system_user).outerjoin(AdminUser, + AdminUser.admin_user == Server.admin_user).filter( + Server.id == i).all() + asset_data = get_server_sysinfo(server_list) + ins_log.read_log('info', '资产信息：{}'.format(asset_data)) + self.update_asset(asset_data) + + +def main(state): + """ + 机器状态,分为:new, true, false + new: 表示新加的机器 + 1. 先推送密钥 + 2. 多进程使用absible获取主机资产信息 + 3. 将资产信息写入/更新到数据库 + + true: 表示已经可以正常免密钥登陆 + 1. 多进程使用absible获取主机资产信息 + 2. 将资产信息写入/更新到数据库 + + false: 表示主机可能配置有问题,无法正常更新资产 + 1. 将错误信息记录数据库 + :param state: 机器的状态 + :return: + """ + + obj = AssetServerAUtoUpdate(state) + if state == 'new': + if not obj.rsync_public_key(): + # 如果没有发现有新增的主机,直接PASS + return + obj.get_host_info() + elif state == 'true': + obj.get_host_info() + + +def new_tail_data(): + main('new') + + +def true_tail_data(): + main('true') + + +if __name__ == '__main__': + fire.Fire(main) diff --git a/libs/server/collect_asset_info.py b/libs/server/collect_asset_info.py index b63bb52..694da0b 100644 --- a/libs/server/collect_asset_info.py +++ b/libs/server/collect_asset_info.py @@ -1,152 +1,152 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# @Time : 2019/5/1 13:48 -# @Author : Fred Yangxiaofei -# @File : collect_asset_info.py -# @Role : 通过Ansible API采集资产信息 - - -from libs.ansibleAPI.runner import Runner -from libs.common import M2human - - -def get_host_info(server_list): - if not isinstance(server_list, list): - raise ValueError() - - for host in server_list: - ip = host[0] - user = host[2] - host_dict = { - "host": host[0], - "port": host[1], - "user": host[2], - } - # print(ip,user) - - runner = Runner( - module_name="setup", - module_args="", - remote_user=user, - # remote_port=2222, - pattern="all", - hosts=host_dict, - timeout=10, - - ) - - result = runner.run() - - if result['dark']: - msg = { - ip: { - 'status': False, - 'msg': result['dark'][ip]['msg'] - } - } - return msg - - else: - asset_data = { - ip: { - 'status': True, - 'msg': '获取资产成功', - } - } # ip为key 数据为value - # print(result['contacted'][ip]) - # 资产信息 - # SN - # print(result['contacted'][ip]) - try: - sn = result['contacted'][ip]['ansible_facts']['ansible_product_serial'] - except KeyError: - sn = 'Null' - # 主机名 - try: - #这个带.没办法获取到 - #host_name = result['contacted'][ip]['ansible_facts']['ansible_hostname'] - host_name = result['contacted'][ip]['ansible_facts']['ansible_fqdn'] - #centos7.6版本下获取hostname fqdn会出现全显示localhost6.localdomain6问题 - if host_name == "localhost6.localdomain6": host_name = result['contacted'][ip]['ansible_facts']['ansible_hostname'] - except KeyError: - host_name = 'Null' - - # cpu型号 - try: - cpu = result['contacted'][ip]['ansible_facts']['ansible_processor'][-1] - except KeyError: - cpu = 'Null' - - # CPU核心数 - try: - cpu_cores = result['contacted'][ip]['ansible_facts']['ansible_processor_vcpus'] - except KeyError: - cpu_cores = 'Null' - - # 物理内存容量 - try: - memory = result['contacted'][ip]['ansible_facts']['ansible_memtotal_mb'] - except KeyError: - memory = 'Null' - - # 磁盘容量 - try: - disk = sum([int(result['contacted'][ip]['ansible_facts']["ansible_devices"][i]["sectors"]) * \ - int(result['contacted'][ip]['ansible_facts']["ansible_devices"][i][ - "sectorsize"]) / 1024 / 1024 / 1024 \ - for i in result['contacted'][ip]['ansible_facts']["ansible_devices"] if - i[0:2] in ("sd", "ss", "vd", "xv")]) - except KeyError: - disk = 'Null' - - # 磁盘mount - # disk_mount = str( - # [{"mount": i["mount"], "size": i["size_total"] / 1024 / 1024 / 1024} for i in result['contacted'][ip]['ansible_facts']["ansible_mounts"]]) - # - # print(disk_mount) - - # 服务器类型 - try: - os_type = " ".join((result['contacted'][ip]['ansible_facts']["ansible_distribution"], - result['contacted'][ip]['ansible_facts']["ansible_distribution_version"])) - except KeyError: - os_type = 'Null' - - try: - - os_kernel = result['contacted'][ip]['ansible_facts']['ansible_kernel'] - except KeyError: - os_kernel = 'Null' - - asset_data[ip]['sn'] = sn - asset_data[ip]['host_name'] = host_name - asset_data[ip]['cpu'] = cpu - asset_data[ip]['cpu_cores'] = cpu_cores - asset_data[ip]['memory'] = M2human(memory) - asset_data[ip]['disk'] = disk - asset_data[ip]['os_type'] = os_type - asset_data[ip]['os_kernel'] = os_kernel - - # print(asset_data) - # print('ok') - return asset_data - - -def get_server_sysinfo(server_list): - return get_host_info(server_list) - - -# def get_server_sysinfo(server_list): -# """ -# 多进程采集机器信息 -# :param server_list: 主机列表 -# :return: -# """ -# #print(list(exec_thread(func=get_host_info, iterable1=server_list))) -# return list(exec_thread(func=get_host_info, iterable1=server_list)) - - -if __name__ == '__main__': - pass -# server_list = [[[('172.16.0.120', 22, 'root')],[('172.16.0.93', 22, 'root')], [('1.1.1.1', 22, 'root')]]] -# get_server_sysinfo(server_list) +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Time : 2019/5/1 13:48 +# @Author : Fred Yangxiaofei +# @File : collect_asset_info.py +# @Role : 通过Ansible API采集资产信息 + + +from libs.ansibleAPI.runner import Runner +from libs.common import M2human + + +def get_host_info(server_list): + if not isinstance(server_list, list): + raise ValueError() + + for host in server_list: + ip = host[0] + user = host[2] + host_dict = { + "host": host[0], + "port": host[1], + "user": host[2], + } + # print(ip,user) + + runner = Runner( + module_name="setup", + module_args="", + remote_user=user, + # remote_port=2222, + pattern="all", + hosts=host_dict, + timeout=10, + + ) + + result = runner.run() + + if result['dark']: + msg = { + ip: { + 'status': False, + 'msg': result['dark'][ip]['msg'] + } + } + return msg + + else: + asset_data = { + ip: { + 'status': True, + 'msg': '获取资产成功', + } + } # ip为key 数据为value + # print(result['contacted'][ip]) + # 资产信息 + # SN + # print(result['contacted'][ip]) + try: + sn = result['contacted'][ip]['ansible_facts']['ansible_product_serial'] + except KeyError: + sn = 'Null' + # 主机名 + try: + #这个带.没办法获取到 + #host_name = result['contacted'][ip]['ansible_facts']['ansible_hostname'] + host_name = result['contacted'][ip]['ansible_facts']['ansible_fqdn'] + #centos7.6版本下获取hostname fqdn会出现全显示localhost6.localdomain6问题 + if host_name == "localhost6.localdomain6": host_name = result['contacted'][ip]['ansible_facts']['ansible_hostname'] + except KeyError: + host_name = 'Null' + + # cpu型号 + try: + cpu = result['contacted'][ip]['ansible_facts']['ansible_processor'][-1] + except KeyError: + cpu = 'Null' + + # CPU核心数 + try: + cpu_cores = result['contacted'][ip]['ansible_facts']['ansible_processor_vcpus'] + except KeyError: + cpu_cores = 'Null' + + # 物理内存容量 + try: + memory = result['contacted'][ip]['ansible_facts']['ansible_memtotal_mb'] + except KeyError: + memory = 'Null' + + # 磁盘容量 + try: + disk = sum([int(result['contacted'][ip]['ansible_facts']["ansible_devices"][i]["sectors"]) * \ + int(result['contacted'][ip]['ansible_facts']["ansible_devices"][i][ + "sectorsize"]) / 1024 / 1024 / 1024 \ + for i in result['contacted'][ip]['ansible_facts']["ansible_devices"] if + i[0:2] in ("sd", "ss", "vd", "xv")]) + except KeyError: + disk = 'Null' + + # 磁盘mount + # disk_mount = str( + # [{"mount": i["mount"], "size": i["size_total"] / 1024 / 1024 / 1024} for i in result['contacted'][ip]['ansible_facts']["ansible_mounts"]]) + # + # print(disk_mount) + + # 服务器类型 + try: + os_type = " ".join((result['contacted'][ip]['ansible_facts']["ansible_distribution"], + result['contacted'][ip]['ansible_facts']["ansible_distribution_version"])) + except KeyError: + os_type = 'Null' + + try: + + os_kernel = result['contacted'][ip]['ansible_facts']['ansible_kernel'] + except KeyError: + os_kernel = 'Null' + + asset_data[ip]['sn'] = sn + asset_data[ip]['host_name'] = host_name + asset_data[ip]['cpu'] = cpu + asset_data[ip]['cpu_cores'] = cpu_cores + asset_data[ip]['memory'] = M2human(memory) + asset_data[ip]['disk'] = disk + asset_data[ip]['os_type'] = os_type + asset_data[ip]['os_kernel'] = os_kernel + + # print(asset_data) + # print('ok') + return asset_data + + +def get_server_sysinfo(server_list): + return get_host_info(server_list) + + +# def get_server_sysinfo(server_list): +# """ +# 多进程采集机器信息 +# :param server_list: 主机列表 +# :return: +# """ +# #print(list(exec_thread(func=get_host_info, iterable1=server_list))) +# return list(exec_thread(func=get_host_info, iterable1=server_list)) + + +if __name__ == '__main__': + pass +# server_list = [[[('172.16.0.120', 22, 'root')],[('172.16.0.93', 22, 'root')], [('1.1.1.1', 22, 'root')]]] +# get_server_sysinfo(server_list) diff --git a/libs/server/push_system_user.py b/libs/server/push_system_user.py index 7fa572a..862e4fd 100644 --- a/libs/server/push_system_user.py +++ b/libs/server/push_system_user.py @@ -1,340 +1,340 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# @Time : 2019/5/7 10:06 -# @Author : Fred Yangxiaofei -# @File : push_system_user.py -# @Role : 多进程推送系统用户，配置免密钥和sudo权限 - - -from libs.ansibleAPI.runner import Runner -from libs.ansibleAPI.myinventory import MyInventory -from libs.db_context import DBContext -from models.server import SystemUser, model_to_dict, Server, AdminUser, AssetErrorLog -from opssdk.operate import MyCryptV2 - - -class PushSystemUser(): - def __init__(self): - self.sudo = 'echo "" | sudo -S' - self.exc_list = ['root', 'ubuntu', 'ec2-user', 'centos', 'admin'] - self.err_msg = {"status": False, "ip": "", "msg": ""} - self.err_list = [] - self.msg = '' - - def run(self, module_name="shell", module_args='', hosts='', remote_user="root", timeout=10, forks=10): - '''Ansible运行函数''' - runner = Runner( - module_name=module_name, - module_args=module_args, - remote_user=remote_user, - pattern="all", - hosts=hosts, - forks=forks, - timeout=timeout, - ) - - result = runner.run() - return result - - def write_error_log(self): - """ - 将错误日志写入数据库 - :return: - """ - # 错误日志记录，更新状态 - with DBContext('w') as session: - for msg in self.err_list: - ip = msg.get('ip') - msg = msg.get('msg') - error_log = '错误信息：{}'.format(msg) - print(error_log) - session.query(Server).filter(Server.ip == ip).update({Server.state: 'false'}) - exist_ip = session.query(AssetErrorLog).filter(AssetErrorLog.ip == ip).first() - if exist_ip: - session.query(AssetErrorLog).filter(AssetErrorLog.ip == ip).update( - {AssetErrorLog.error_log: error_log}) - else: - new_error_log = AssetErrorLog(ip=ip, error_log=error_log) - session.add(new_error_log) - session.commit() - - def get_system_user(self): - """ - 获取所有系统用户 - :return: - """ - with DBContext('r') as session: - system_user_list = [] - system_user_data = session.query(SystemUser).all() - for data in system_user_data: - data_dict = model_to_dict(data) - data_dict['create_time'] = str(data_dict['create_time']) - data_dict['update_time'] = str(data_dict['update_time']) - system_user_list.append(data_dict) - - return system_user_list - - def get_asset_info(self): - """ - 获取所有可连接资产信息 - :return: - """ - - with DBContext('r') as session: - # 只拿到登陆用到的IP Port User - server_list = session.query(Server.ip, Server.port, AdminUser.system_user, - ).outerjoin(AdminUser, - AdminUser.admin_user == Server.admin_user).filter( - Server.state == 'true').all() - return server_list - - def create_system_user(self): - """ - Ansible API创建系统用户 - :return: - """ - system_user_list = self.get_system_user() - connect_server_list = self.get_asset_info() - # print(connect_server_list) - # connect_server_list = [('172.16.0.93', 22, 'yanghongfei'), ('172.16.0.219', 22, 'root'), - # ('2.2.2.2', 22, 'root')] - for host in connect_server_list: - ip = host[0] - user = host[2] - for data in system_user_list: - system_user = data.get('system_user') - - if system_user in self.exc_list: - self.msg = '{}内置用户不能创建，请跳过此类用户：{}'.format(system_user, self.exc_list) - return self.msg - - bash_shell = data.get('bash_shell') - - module_args = '{sudo} grep -c {system_user} /etc/passwd >> /dev/null || {sudo} useradd {system_user} -s {bash_shell}; echo ok'.format( - sudo=self.sudo, system_user=system_user, bash_shell=bash_shell - ) - result = self.run("shell", module_args, ip, user) - print(result) - - if result['dark']: - self.err_msg = {"status": False, "ip": ip, "msg": result['dark'][ip]['msg']} - self.err_list.append(self.err_msg) - - if self.err_list: - self.write_error_log() - - def configure_keyless(self): - """ - 配置系统用户免密钥登陆 - :return: - """ - # connect_server_list = [('172.16.0.93', 22, 'root'), ('172.16.0.219', 22, 'root'), - # ('2.2.2.2', 22, 'root')] - - connect_server_list = self.get_asset_info() - system_user_list = self.get_system_user() - - for host in connect_server_list: - ip = host[0] - user = host[2] - for data in system_user_list: - mc = MyCryptV2() - system_user = data.get('system_user') - _public_key = mc.my_decrypt(data.get('id_rsa_pub')) # 解密 - - - module_args = '{sudo} [ ! -d /home/{system_user}/.ssh ]&& ' \ - '{sudo} mkdir /home/{system_user}/.ssh &&' \ - '{sudo} chmod 700 /home/{system_user}/.ssh ; ' \ - '{sudo} [ ! -f /home/{system_user}/.ssh/authorized_keys ]&& ' \ - '{sudo} touch /home/{system_user}/.ssh/authorized_keys; ' \ - '{sudo} chown -R {system_user}.{system_user} /home/{system_user}/.ssh ; ' \ - '{sudo} grep -c "{public_key}" /home/{system_user}/.ssh/authorized_keys >> /dev/null && ' \ - 'echo "is exist public_key, auto pass...." || ' \ - '{sudo} echo "{public_key}" >> /home/{system_user}/.ssh/authorized_keys && ' \ - '{sudo} chmod 600 /home/{system_user}/.ssh/authorized_keys && ' \ - 'echo ok'.format(sudo=self.sudo, system_user=system_user, public_key=_public_key) - # print(module_args) - - result = self.run("shell", module_args, ip, user) - - print(result) - if result['dark']: - self.err_msg = {"status": False, "ip": ip, "msg": result['dark'][ip]['msg']} - self.err_list.append(self.err_msg) - - if self.err_list: - self.write_error_log() - - def configure_sudoers(self): - """ - 配置sudo权限 - :return: - """ - # connect_server_list = [('172.16.0.93', 22, 'yanghongfei'), ('172.16.0.219', 22, 'root')] - connect_server_list = self.get_asset_info() - system_user_list = self.get_system_user() - - for host in connect_server_list: - ip = host[0] - user = host[2] - - # if user != 'root': - # print('配置sudo权限必须是ROOT才可以操作') - # return False - for data in system_user_list: - system_user = data.get('system_user') - if system_user == 'root': - print('root属于系统内置用户，不能作为系统用户来推送') - return - - sudo_list = data.get('sudo_list') - # 这里还缺少一个update要写，比如用户在前端修改了sudo内容，再次推送进来要更新进去。 - module_args = "{sudo} grep -r ^{system_user} /etc/sudoers && echo 'is exist sudoers, auto pass... ' || {sudo} sed -i '$a\{system_user} ALL\=(ALL) NOPASSWD: {sudo_list}' /etc/sudoers".format( - sudo=self.sudo, system_user=system_user, sudo_list=sudo_list) - # print(module_args) - result = self.run("shell", module_args, ip, user) - print(result) - if result['dark']: - self.err_msg = {"status": False, "ip": ip, "msg": result['dark'][ip]['msg']} - self.err_list.append(self.err_msg) - if self.err_list: - self.write_error_log() - - def update_user_sudo(self, system_user, sudo_list): - """ - 配置系统用户的sudoers，先删除，再添加 - :return: - """ - # connect_server_list = [('172.16.0.93', 22, 'yanghongfei'), ('172.16.0.219', 22, 'root')] - - if not self.delete_user_sudo(system_user): - print('删除用户sudo失败') - return False - connect_server_list = self.get_asset_info() - for host in connect_server_list: - ip = host[0] - user = host[2] - - if system_user == 'root': - print('root属于系统内置用户，不能作为系统用户来推送') - return False - - module_args = "{sudo} grep -r ^{system_user} /etc/sudoers && echo 'is exist sudoers, auto pass... ' || {sudo} sed -i '$a\{system_user} ALL\=(ALL) NOPASSWD: {sudo_list}' /etc/sudoers".format( - sudo=self.sudo, system_user=system_user, sudo_list=sudo_list) - # print(module_args) - result = self.run("shell", module_args, ip, user) - print(result) - if result['dark']: - self.err_msg = {"status": False, "ip": ip, "msg": result['dark'][ip]['msg']} - self.err_list.append(self.err_msg) - if self.err_list: - self.write_error_log() - - def delete_system_user(self, system_user): - """ - 删除推送的系统用户，排除常见的用户：root ubuntu ec2-user centos admin等 - :return: - """ - - if system_user == 'root': - msg = 'root用户不能被删除' - print(msg) - return msg - - exc_list = ['ubuntu', 'ec2-user', 'centos', 'admin'] - if system_user in exc_list: - msg = '{}属于系统内置/危险用户，不能被删除'.format(system_user) - print(msg) - return msg - - print('start del user') - connect_server_list = self.get_asset_info() - # connect_server_list = [('172.16.0.93', 22, 'yanghongfei')] - - for host in connect_server_list: - ip = host[0] - user = host[2] - if system_user == user: - msg = '删除的用户不能是自己(管理用户)' - print(msg) - return msg - - module_args = '{sudo} grep -c "{system_user}" /etc/passwd >> /dev/null && {sudo} userdel -r {system_user} || echo "user is not exist, pass"'.format( - sudo=self.sudo, system_user=system_user) # 存在就删除 - result = self.run("shell", module_args, ip, user) - print(result) - if result['dark']: - self.err_msg = {"status": False, "ip": ip, "msg": result['dark'][ip]['msg']} - self.err_list.append(self.err_msg) - if self.err_list: - self.write_error_log() - return self.err_list - - def delete_user_sudo(self, system_user): - """ - 删除用户的sudo权限 - :return: - """ - if system_user == 'root': - msg = 'root用户不能被删除' - print(msg) - - return msg - - exc_list = ['ubuntu', 'ec2-user', 'centos', 'admin'] - if system_user in exc_list: - msg = '{}属于系统内置/危险用户，不能被删除'.format(system_user) - print(msg) - return msg - - print('start del user') - connect_server_list = [('172.16.0.93', 22, 'yanghongfei')] - - for host in connect_server_list: - ip = host[0] - user = host[2] - if system_user == user: - msg = '删除的用户不能是自己(管理用户)' - print(msg) - return msg - - module_args = "{sudo} sed -i 's/^{system_user}.*//' /etc/sudoers".format(sudo=self.sudo, - system_user=system_user) - print(module_args) - result = self.run("shell", module_args, ip, user) - print(result) - if result['dark']: - self.err_msg = {"status": False, "ip": ip, "msg": result['dark'][ip]['msg']} - self.err_list.append(self.err_msg) - if self.err_list: - self.write_error_log() - return self.err_list - - # def add_user(self): - # connect_server_list = [('172.16.0.93', 22, 'root'), ('2.2.2.2', 22, 'root')] - # for host in connect_server_list: - # ip = host[0] - # user = host[2] - # module_args1 = 'echo "" | sudo -S cat /etc/sudoers' - # module_args = 'name={} shell=/bin/bash'.format('testuser') - # result = self.run("shell", module_args,ip,user) - # - # print(result) - - -def main(): - """ - 这是二期规划，跳板用的 - :return: - """ - obj = PushSystemUser() - # obj.delete_system_user('sudo_test') - # obj.delete_user_sudo('sudo_test') - obj.create_system_user() - obj.configure_keyless() - obj.configure_sudoers() - - -if __name__ == '__main__': - main() +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Time : 2019/5/7 10:06 +# @Author : Fred Yangxiaofei +# @File : push_system_user.py +# @Role : 多进程推送系统用户，配置免密钥和sudo权限 + + +from libs.ansibleAPI.runner import Runner +from libs.ansibleAPI.myinventory import MyInventory +from libs.db_context import DBContext +from models.server import SystemUser, model_to_dict, Server, AdminUser, AssetErrorLog +from opssdk.operate import MyCryptV2 + + +class PushSystemUser(): + def __init__(self): + self.sudo = 'echo "" | sudo -S' + self.exc_list = ['root', 'ubuntu', 'ec2-user', 'centos', 'admin'] + self.err_msg = {"status": False, "ip": "", "msg": ""} + self.err_list = [] + self.msg = '' + + def run(self, module_name="shell", module_args='', hosts='', remote_user="root", timeout=10, forks=10): + '''Ansible运行函数''' + runner = Runner( + module_name=module_name, + module_args=module_args, + remote_user=remote_user, + pattern="all", + hosts=hosts, + forks=forks, + timeout=timeout, + ) + + result = runner.run() + return result + + def write_error_log(self): + """ + 将错误日志写入数据库 + :return: + """ + # 错误日志记录，更新状态 + with DBContext('w') as session: + for msg in self.err_list: + ip = msg.get('ip') + msg = msg.get('msg') + error_log = '错误信息：{}'.format(msg) + print(error_log) + session.query(Server).filter(Server.ip == ip).update({Server.state: 'false'}) + exist_ip = session.query(AssetErrorLog).filter(AssetErrorLog.ip == ip).first() + if exist_ip: + session.query(AssetErrorLog).filter(AssetErrorLog.ip == ip).update( + {AssetErrorLog.error_log: error_log}) + else: + new_error_log = AssetErrorLog(ip=ip, error_log=error_log) + session.add(new_error_log) + session.commit() + + def get_system_user(self): + """ + 获取所有系统用户 + :return: + """ + with DBContext('r') as session: + system_user_list = [] + system_user_data = session.query(SystemUser).all() + for data in system_user_data: + data_dict = model_to_dict(data) + data_dict['create_time'] = str(data_dict['create_time']) + data_dict['update_time'] = str(data_dict['update_time']) + system_user_list.append(data_dict) + + return system_user_list + + def get_asset_info(self): + """ + 获取所有可连接资产信息 + :return: + """ + + with DBContext('r') as session: + # 只拿到登陆用到的IP Port User + server_list = session.query(Server.ip, Server.port, AdminUser.system_user, + ).outerjoin(AdminUser, + AdminUser.admin_user == Server.admin_user).filter( + Server.state == 'true').all() + return server_list + + def create_system_user(self): + """ + Ansible API创建系统用户 + :return: + """ + system_user_list = self.get_system_user() + connect_server_list = self.get_asset_info() + # print(connect_server_list) + # connect_server_list = [('172.16.0.93', 22, 'yanghongfei'), ('172.16.0.219', 22, 'root'), + # ('2.2.2.2', 22, 'root')] + for host in connect_server_list: + ip = host[0] + user = host[2] + for data in system_user_list: + system_user = data.get('system_user') + + if system_user in self.exc_list: + self.msg = '{}内置用户不能创建，请跳过此类用户：{}'.format(system_user, self.exc_list) + return self.msg + + bash_shell = data.get('bash_shell') + + module_args = '{sudo} grep -c {system_user} /etc/passwd >> /dev/null || {sudo} useradd {system_user} -s {bash_shell}; echo ok'.format( + sudo=self.sudo, system_user=system_user, bash_shell=bash_shell + ) + result = self.run("shell", module_args, ip, user) + print(result) + + if result['dark']: + self.err_msg = {"status": False, "ip": ip, "msg": result['dark'][ip]['msg']} + self.err_list.append(self.err_msg) + + if self.err_list: + self.write_error_log() + + def configure_keyless(self): + """ + 配置系统用户免密钥登陆 + :return: + """ + # connect_server_list = [('172.16.0.93', 22, 'root'), ('172.16.0.219', 22, 'root'), + # ('2.2.2.2', 22, 'root')] + + connect_server_list = self.get_asset_info() + system_user_list = self.get_system_user() + + for host in connect_server_list: + ip = host[0] + user = host[2] + for data in system_user_list: + mc = MyCryptV2() + system_user = data.get('system_user') + _public_key = mc.my_decrypt(data.get('id_rsa_pub')) # 解密 + + + module_args = '{sudo} [ ! -d /home/{system_user}/.ssh ]&& ' \ + '{sudo} mkdir /home/{system_user}/.ssh &&' \ + '{sudo} chmod 700 /home/{system_user}/.ssh ; ' \ + '{sudo} [ ! -f /home/{system_user}/.ssh/authorized_keys ]&& ' \ + '{sudo} touch /home/{system_user}/.ssh/authorized_keys; ' \ + '{sudo} chown -R {system_user}.{system_user} /home/{system_user}/.ssh ; ' \ + '{sudo} grep -c "{public_key}" /home/{system_user}/.ssh/authorized_keys >> /dev/null && ' \ + 'echo "is exist public_key, auto pass...." || ' \ + '{sudo} echo "{public_key}" >> /home/{system_user}/.ssh/authorized_keys && ' \ + '{sudo} chmod 600 /home/{system_user}/.ssh/authorized_keys && ' \ + 'echo ok'.format(sudo=self.sudo, system_user=system_user, public_key=_public_key) + # print(module_args) + + result = self.run("shell", module_args, ip, user) + + print(result) + if result['dark']: + self.err_msg = {"status": False, "ip": ip, "msg": result['dark'][ip]['msg']} + self.err_list.append(self.err_msg) + + if self.err_list: + self.write_error_log() + + def configure_sudoers(self): + """ + 配置sudo权限 + :return: + """ + # connect_server_list = [('172.16.0.93', 22, 'yanghongfei'), ('172.16.0.219', 22, 'root')] + connect_server_list = self.get_asset_info() + system_user_list = self.get_system_user() + + for host in connect_server_list: + ip = host[0] + user = host[2] + + # if user != 'root': + # print('配置sudo权限必须是ROOT才可以操作') + # return False + for data in system_user_list: + system_user = data.get('system_user') + if system_user == 'root': + print('root属于系统内置用户，不能作为系统用户来推送') + return + + sudo_list = data.get('sudo_list') + # 这里还缺少一个update要写，比如用户在前端修改了sudo内容，再次推送进来要更新进去。 + module_args = "{sudo} grep -r ^{system_user} /etc/sudoers && echo 'is exist sudoers, auto pass... ' || {sudo} sed -i '$a\{system_user} ALL\=(ALL) NOPASSWD: {sudo_list}' /etc/sudoers".format( + sudo=self.sudo, system_user=system_user, sudo_list=sudo_list) + # print(module_args) + result = self.run("shell", module_args, ip, user) + print(result) + if result['dark']: + self.err_msg = {"status": False, "ip": ip, "msg": result['dark'][ip]['msg']} + self.err_list.append(self.err_msg) + if self.err_list: + self.write_error_log() + + def update_user_sudo(self, system_user, sudo_list): + """ + 配置系统用户的sudoers，先删除，再添加 + :return: + """ + # connect_server_list = [('172.16.0.93', 22, 'yanghongfei'), ('172.16.0.219', 22, 'root')] + + if not self.delete_user_sudo(system_user): + print('删除用户sudo失败') + return False + connect_server_list = self.get_asset_info() + for host in connect_server_list: + ip = host[0] + user = host[2] + + if system_user == 'root': + print('root属于系统内置用户，不能作为系统用户来推送') + return False + + module_args = "{sudo} grep -r ^{system_user} /etc/sudoers && echo 'is exist sudoers, auto pass... ' || {sudo} sed -i '$a\{system_user} ALL\=(ALL) NOPASSWD: {sudo_list}' /etc/sudoers".format( + sudo=self.sudo, system_user=system_user, sudo_list=sudo_list) + # print(module_args) + result = self.run("shell", module_args, ip, user) + print(result) + if result['dark']: + self.err_msg = {"status": False, "ip": ip, "msg": result['dark'][ip]['msg']} + self.err_list.append(self.err_msg) + if self.err_list: + self.write_error_log() + + def delete_system_user(self, system_user): + """ + 删除推送的系统用户，排除常见的用户：root ubuntu ec2-user centos admin等 + :return: + """ + + if system_user == 'root': + msg = 'root用户不能被删除' + print(msg) + return msg + + exc_list = ['ubuntu', 'ec2-user', 'centos', 'admin'] + if system_user in exc_list: + msg = '{}属于系统内置/危险用户，不能被删除'.format(system_user) + print(msg) + return msg + + print('start del user') + connect_server_list = self.get_asset_info() + # connect_server_list = [('172.16.0.93', 22, 'yanghongfei')] + + for host in connect_server_list: + ip = host[0] + user = host[2] + if system_user == user: + msg = '删除的用户不能是自己(管理用户)' + print(msg) + return msg + + module_args = '{sudo} grep -c "{system_user}" /etc/passwd >> /dev/null && {sudo} userdel -r {system_user} || echo "user is not exist, pass"'.format( + sudo=self.sudo, system_user=system_user) # 存在就删除 + result = self.run("shell", module_args, ip, user) + print(result) + if result['dark']: + self.err_msg = {"status": False, "ip": ip, "msg": result['dark'][ip]['msg']} + self.err_list.append(self.err_msg) + if self.err_list: + self.write_error_log() + return self.err_list + + def delete_user_sudo(self, system_user): + """ + 删除用户的sudo权限 + :return: + """ + if system_user == 'root': + msg = 'root用户不能被删除' + print(msg) + + return msg + + exc_list = ['ubuntu', 'ec2-user', 'centos', 'admin'] + if system_user in exc_list: + msg = '{}属于系统内置/危险用户，不能被删除'.format(system_user) + print(msg) + return msg + + print('start del user') + connect_server_list = [('172.16.0.93', 22, 'yanghongfei')] + + for host in connect_server_list: + ip = host[0] + user = host[2] + if system_user == user: + msg = '删除的用户不能是自己(管理用户)' + print(msg) + return msg + + module_args = "{sudo} sed -i 's/^{system_user}.*//' /etc/sudoers".format(sudo=self.sudo, + system_user=system_user) + print(module_args) + result = self.run("shell", module_args, ip, user) + print(result) + if result['dark']: + self.err_msg = {"status": False, "ip": ip, "msg": result['dark'][ip]['msg']} + self.err_list.append(self.err_msg) + if self.err_list: + self.write_error_log() + return self.err_list + + # def add_user(self): + # connect_server_list = [('172.16.0.93', 22, 'root'), ('2.2.2.2', 22, 'root')] + # for host in connect_server_list: + # ip = host[0] + # user = host[2] + # module_args1 = 'echo "" | sudo -S cat /etc/sudoers' + # module_args = 'name={} shell=/bin/bash'.format('testuser') + # result = self.run("shell", module_args,ip,user) + # + # print(result) + + +def main(): + """ + 这是二期规划，跳板用的 + :return: + """ + obj = PushSystemUser() + # obj.delete_system_user('sudo_test') + # obj.delete_user_sudo('sudo_test') + obj.create_system_user() + obj.configure_keyless() + obj.configure_sudoers() + + +if __name__ == '__main__': + main() diff --git a/libs/server/server_common.py b/libs/server/server_common.py index f1b28fc..c646039 100644 --- a/libs/server/server_common.py +++ b/libs/server/server_common.py @@ -1,123 +1,123 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# @Time : 2019/5/15 14:44 -# @Author : Fred Yangxiaofei -# @File : server_common.py -# @Role : server公用方法，记录日志，更新资产,推送密钥，主要给手动更新资产使用 - -from models.server import Server, AssetErrorLog, ServerDetail -from libs.db_context import DBContext -from libs.web_logs import ins_log -from libs.server.sync_public_key import RsyncPublicKey, start_rsync -import sqlalchemy - - -def write_error_log(error_list): - with DBContext('w') as session: - for i in error_list: - ip = i.get('ip') - msg = i.get('msg') - error_log = '推送公钥失败, 错误信息：{}'.format(msg) - ins_log.read_log('error', error_log) - session.query(Server).filter(Server.ip == ip).update({Server.state: 'false'}) - exist_ip = session.query(AssetErrorLog).filter(AssetErrorLog.ip == ip).first() - if exist_ip: - session.query(AssetErrorLog).filter(AssetErrorLog.ip == ip).update( - {AssetErrorLog.error_log: error_log}) - else: - new_error_log = AssetErrorLog(ip=ip, error_log=error_log) - session.add(new_error_log) - session.commit() - - -def update_asset(asset_data): - """ - 更新资产到数据库 - :param host_data: 主机返回的资产采集基础数据 - :return: - """ - with DBContext('w') as session: - for k, v in asset_data.items(): - try: - if asset_data[k].get('status'): - _sn = v.get('sn', None) - _hostname = v.get('host_name', None) - _cpu = v.get('cpu', None) - _cpu_cores = v.get('cpu_cores', None) - _memory = v.get('memory', None) - _disk = v.get('disk', None) - _os_type = v.get('os_type', None) - _os_kernel = v.get('os_kernel', None) - # _instance_id = v.get('instance_id', None) - # _instance_type = v.get('instance_type', None) - # _instance_state = v.get('instance_state', None) - - exist_detail = session.query(ServerDetail).filter(ServerDetail.ip == k).first() - if not exist_detail: - # 不存在就新建 - new_server_detail = ServerDetail(ip=k, sn=_sn, cpu=_cpu, cpu_cores=_cpu_cores, - memory=_memory, disk=_disk, - os_type=_os_type, os_kernel=_os_kernel) - session.add(new_server_detail) - session.commit() - session.query(Server).filter(Server.ip == k).update( - {Server.hostname: _hostname, Server.state: 'true'}) - session.commit() - else: - # 存在就更新 - session.query(ServerDetail).filter(ServerDetail.ip == k).update({ - ServerDetail.sn: _sn, ServerDetail.ip: k, - ServerDetail.cpu: _cpu, ServerDetail.cpu_cores: _cpu_cores, - ServerDetail.disk: _disk, ServerDetail.memory: _memory, - ServerDetail.os_type: _os_type, ServerDetail.os_kernel: _os_kernel, - }) - - session.query(Server).filter(Server.ip == k).update( - {Server.hostname: _hostname, Server.state: 'true'}) - session.commit() - except sqlalchemy.exc.IntegrityError as e: - ins_log.read_log('error', e) - # 状态改为Flse->删除主机Detail--记录错误信息 - session.query(Server).filter(Server.ip == k).update({Server.state: 'false'}) - session.query(ServerDetail).filter(ServerDetail.ip == k).delete( - synchronize_session=False) - - exist_ip = session.query(AssetErrorLog).filter(AssetErrorLog.ip == k).first() - error_log = str(e) - if exist_ip: - session.query(AssetErrorLog).filter(AssetErrorLog.ip == k).update( - {AssetErrorLog.error_log: error_log}) - else: - new_error_log = AssetErrorLog(ip=k, error_log=error_log) - session.add(new_error_log) - - session.commit() - return False - - -def rsync_public_key(server_list): - """ - 推送PublicKey - :return: 只返回推送成功的，失败的直接写错误日志 - """ - # server_list = [('47.100.231.147', 22, 'root', '-----BEGIN RSA PRIVATE KEYxxxxxEND RSA PRIVATE KEY-----', 'false')] - ins_log.read_log('info', 'rsync public key to server') - rsync_error_list = [] - rsync_sucess_list = [] - sync_key_obj = RsyncPublicKey() - check = sync_key_obj.check_rsa() - if check: - res_data = start_rsync(server_list) - if not res_data.get('status'): - rsync_error_list.append(res_data) - else: - rsync_sucess_list.append(res_data) - - if rsync_error_list: - write_error_log(rsync_error_list) - - return rsync_sucess_list - - -if __name__ == '__main__': - pass +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Time : 2019/5/15 14:44 +# @Author : Fred Yangxiaofei +# @File : server_common.py +# @Role : server公用方法，记录日志，更新资产,推送密钥，主要给手动更新资产使用 + +from models.server import Server, AssetErrorLog, ServerDetail +from libs.db_context import DBContext +from libs.web_logs import ins_log +from libs.server.sync_public_key import RsyncPublicKey, start_rsync +import sqlalchemy + + +def write_error_log(error_list): + with DBContext('w') as session: + for i in error_list: + ip = i.get('ip') + msg = i.get('msg') + error_log = '推送公钥失败, 错误信息：{}'.format(msg) + ins_log.read_log('error', error_log) + session.query(Server).filter(Server.ip == ip).update({Server.state: 'false'}) + exist_ip = session.query(AssetErrorLog).filter(AssetErrorLog.ip == ip).first() + if exist_ip: + session.query(AssetErrorLog).filter(AssetErrorLog.ip == ip).update( + {AssetErrorLog.error_log: error_log}) + else: + new_error_log = AssetErrorLog(ip=ip, error_log=error_log) + session.add(new_error_log) + session.commit() + + +def update_asset(asset_data): + """ + 更新资产到数据库 + :param host_data: 主机返回的资产采集基础数据 + :return: + """ + with DBContext('w') as session: + for k, v in asset_data.items(): + try: + if asset_data[k].get('status'): + _sn = v.get('sn', None) + _hostname = v.get('host_name', None) + _cpu = v.get('cpu', None) + _cpu_cores = v.get('cpu_cores', None) + _memory = v.get('memory', None) + _disk = v.get('disk', None) + _os_type = v.get('os_type', None) + _os_kernel = v.get('os_kernel', None) + # _instance_id = v.get('instance_id', None) + # _instance_type = v.get('instance_type', None) + # _instance_state = v.get('instance_state', None) + + exist_detail = session.query(ServerDetail).filter(ServerDetail.ip == k).first() + if not exist_detail: + # 不存在就新建 + new_server_detail = ServerDetail(ip=k, sn=_sn, cpu=_cpu, cpu_cores=_cpu_cores, + memory=_memory, disk=_disk, + os_type=_os_type, os_kernel=_os_kernel) + session.add(new_server_detail) + session.commit() + session.query(Server).filter(Server.ip == k).update( + {Server.hostname: _hostname, Server.state: 'true'}) + session.commit() + else: + # 存在就更新 + session.query(ServerDetail).filter(ServerDetail.ip == k).update({ + ServerDetail.sn: _sn, ServerDetail.ip: k, + ServerDetail.cpu: _cpu, ServerDetail.cpu_cores: _cpu_cores, + ServerDetail.disk: _disk, ServerDetail.memory: _memory, + ServerDetail.os_type: _os_type, ServerDetail.os_kernel: _os_kernel, + }) + + session.query(Server).filter(Server.ip == k).update( + {Server.hostname: _hostname, Server.state: 'true'}) + session.commit() + except sqlalchemy.exc.IntegrityError as e: + ins_log.read_log('error', e) + # 状态改为Flse->删除主机Detail--记录错误信息 + session.query(Server).filter(Server.ip == k).update({Server.state: 'false'}) + session.query(ServerDetail).filter(ServerDetail.ip == k).delete( + synchronize_session=False) + + exist_ip = session.query(AssetErrorLog).filter(AssetErrorLog.ip == k).first() + error_log = str(e) + if exist_ip: + session.query(AssetErrorLog).filter(AssetErrorLog.ip == k).update( + {AssetErrorLog.error_log: error_log}) + else: + new_error_log = AssetErrorLog(ip=k, error_log=error_log) + session.add(new_error_log) + + session.commit() + return False + + +def rsync_public_key(server_list): + """ + 推送PublicKey + :return: 只返回推送成功的，失败的直接写错误日志 + """ + # server_list = [('47.100.231.147', 22, 'root', '-----BEGIN RSA PRIVATE KEYxxxxxEND RSA PRIVATE KEY-----', 'false')] + ins_log.read_log('info', 'rsync public key to server') + rsync_error_list = [] + rsync_sucess_list = [] + sync_key_obj = RsyncPublicKey() + check = sync_key_obj.check_rsa() + if check: + res_data = start_rsync(server_list) + if not res_data.get('status'): + rsync_error_list.append(res_data) + else: + rsync_sucess_list.append(res_data) + + if rsync_error_list: + write_error_log(rsync_error_list) + + return rsync_sucess_list + + +if __name__ == '__main__': + pass diff --git a/libs/server/sync_public_key.py b/libs/server/sync_public_key.py index 66875f1..ef184f0 100644 --- a/libs/server/sync_public_key.py +++ b/libs/server/sync_public_key.py @@ -1,156 +1,156 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# @Time : 2019/5/1 13:28 -# @Author : Fred Yangxiaofei -# @File : sync_public_key.py -# @Role : 推送公钥到主机，实现免密钥登陆 - -import os -import paramiko -from settings import PUBLIC_KEY -from models.server import SSHConfigs -from libs.db_context import DBContext -from libs.web_logs import ins_log -from libs.common import remote_upload_file, get_key_file, exec_shell -import fire - -BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - - -class RsyncPublicKey(): - def __init__(self): - self.msg = '' - - def check_public_path(self): - """ - 检查是否有这个目录，没有目录就新建 - :return: - """ - PUBLIC_KEY_PATH = os.path.dirname(PUBLIC_KEY) - cmd = '[ ! -d {} ] && mkdir {} && chmod 700 {} ; '.format(PUBLIC_KEY_PATH, PUBLIC_KEY_PATH, PUBLIC_KEY_PATH) - code, ret = exec_shell(cmd) - if code == 0: - return True - else: - return False - - def init_rsa(self): - '''Server端生成秘钥对''' - cmd = 'ssh-keygen -t rsa -P "" -f {}/id_rsa'.format(os.path.dirname((PUBLIC_KEY))) - - code, ret = exec_shell(cmd) - if code == 0: - return True - else: - return False - - def save_of(self): - '''把读取或者生成的秘钥信息保存到SQL''' - with open('%s/id_rsa' % os.path.dirname(PUBLIC_KEY), 'r') as id_rsa, open( - '%s/id_rsa.pub' % os.path.dirname(PUBLIC_KEY), 'r') as id_rsa_pub: - with DBContext('w') as session: - new_config = SSHConfigs(name='cmdb', id_rsa=id_rsa.read(), id_rsa_pub=id_rsa_pub.read()) - session.add(new_config) - session.commit() - - def check_rsa(self): - """ - 检查CMDB 密钥配置,没有则创建新的写入数据库 - :return: - """ - # 检查路径 - self.check_public_path() - with DBContext('r') as session: - # 这张表里面直有一条信息，名字：cmdb， 一对密钥 - exist_rsa = session.query(SSHConfigs.id, SSHConfigs.id_rsa_pub, SSHConfigs.id_rsa).filter( - SSHConfigs.name == 'cmdb').first() - if not exist_rsa: - # 检查本地是否存在 - local_id_rsa_exist = os.path.exists('{}/id_rsa'.format(os.path.dirname(PUBLIC_KEY))) - if not local_id_rsa_exist: - check_rsa = self.init_rsa() - if not check_rsa: - return False - self.save_of() - - elif exist_rsa: - PUBLIC_KEY_PATH = os.path.dirname(PUBLIC_KEY) - id_rsa_pub = exist_rsa[1] - id_rsa = exist_rsa[2] - cmd1 = 'echo "{}" > {}/id_rsa.pub'.format(id_rsa_pub, PUBLIC_KEY_PATH) - cmd2 = 'echo "{}" > {}/id_rsa && chmod 600 {}/id_rsa'.format(id_rsa, PUBLIC_KEY_PATH, PUBLIC_KEY_PATH) - exec_shell(cmd1) - exec_shell(cmd2) - return True - else: - return True - - def sync_key(self, server_list): - """ - 批量下发server端公钥到client端 - :param server_list: 主机信息，IP端口用户密码 - :return: - """ - if not isinstance(server_list, list): - raise ValueError() - - ip = server_list[0][0] - port = server_list[0][1] - user = server_list[0][2] - user_key = server_list[0][3] - cmd = '[ ! -d ~/.ssh ] && mkdir ~/.ssh && chmod 700 ~/.ssh ; ' \ - '[ ! -f ~/.ssh/authorized_keys ] && touch ~/.ssh/authorized_keys; ' \ - 'grep -c "`cat /tmp/id_rsa.pub`" ~/.ssh/authorized_keys >> /dev/null;' \ - '[ $? == 0 ] || cat /tmp/id_rsa.pub >> ~/.ssh/authorized_keys && chmod 600 ~/.ssh/authorized_keys && echo ok' - # 将key写到本地 - ssh_key_file = get_key_file(user_key) - if ssh_key_file: - try: - res = remote_upload_file(ip, user, ssh_key_file, cmd, PUBLIC_KEY, '/tmp/id_rsa.pub', port) - if res[0] == 'ok': - self.msg = { - 'status': True, - 'ip': ip, - 'port': port, - 'user': user, - 'msg': '推送成功' - } - else: - # 状态改为False - self.msg = { - 'status': False, - 'ip': ip, - 'msg': '推送失败' - } - except paramiko.ssh_exception.AuthenticationException: - self.msg = { - 'status': False, - 'ip': ip, - 'msg': '认证失败，请检查管理用户Key是否正确' - } - - except Exception as e: - print(e) - self.msg = { - 'status': False, - 'ip': ip, - 'msg': '{}'.format(e) - } - - ins_log.read_log('info', self.msg) - return self.msg - - -def start_rsync(server_list): - """ - 推送CMDB公钥 - # server_list = [('172.16.0.120', 22, 'root', '-----BEGIN RSA PRIVATE KEY-----\nMIIEogIBAAKCAQEAlTbeMPz7G69rRtpAdlcR/YIKd17oFJBcQ6+2ELTs9tA5Tc/W\nZo3+ftex8pKXx9sGyH28ynrwBiJTRmyaSNFssIxDXGbS9mbliLF+3NZX8sE8soUZ\n1bJYhF3VmKhziRgcRr4QKMltVNMFI/p8ofdsrbNFX6f2OA0iP3bHTbTUzgDiRjwx\nubhXolog/eZboCTahKClPlT3ffhoyYihLX8zjgAxJsFpUAFfVXUrGjD+ttEPoSjg\nxhJRdfXMHF9QLmzMwyxJvvreKtXM+LzyrMRcdAedt3mF+BWc89XhqglM5IcR6fIm\nRyXf3dn7mgWm42xGILSOT6qSjJ1JyptkVc7/5wIBIwKCAQBMvSHQDRNtUua2uWL7\nCDUjSjFEtHdbCGn27KbGw3KNj6hxKQ8tbZjThN8sS2QAYgN9DXcJDAZpjf7xBKce\nIo+rQNIvhVaNWX1cPga6t2BuN35NLoJQpN0Q38zvbJMwkA6KyCWRUaXxHBFFtAWV\nIClDY4LDevOnzD1/uW3QMRz8Oi7/Awrce0QX42O8EkKXwf7YtV0ZIfRiKxaNKJVS\nWqxkVCxXdcAwpq6E4zaEXmBTISc2QhFD1n6yS9Z74rSAF2fIVErbBHAGRHMcUpdu\nR3ALKxo4/6R0KQ3rRj/SB8aMPp7KrR2ebSYWAlKDxete66MjIfBXQ1sr/the7iqV\nUJzLAoGBAMQgHreTPrfxtVj2hfpMl+GBvr/VyOoBVobJOZLi4/docuX2fpGVltGb\nb7Lq6F+Wd06nmmHqHhaBTgv2Vw9c8W8svhMarLs96EF/E53p7Xy3f2SmOOUoFbXO\nkMkRxPDK3uEOjDfvptiSuneaDpkbt7B9h+ipaDaTUunSYdgy4oRRAoGBAMLEex3r\nifCeo4YZrqbzk8JdgT6bO869v+DRY7MOB163zS3biIyxJnNF2eODO8diQ/S6uLKf\nKH19hhDfz3ARTdrmPBbd/mWLaHLClvVsaGlt7U0wpOEKk6FqA5x/3+hB2RiKMcL4\nxz08xaqqZ/0A6mal+OCKzZx5E7s/o1XHs0q3AoGBALjrFaXFWGRKS+YqQ869Pr6t\nh/azoC0XNFMywUFZlRx4bFUqPNJoeEH49Embp+UY1uPCmN/5/xyIi2pdSsVXoc84\n0Hhpm4v4ioblh4ZC9dv9eB0RwJ2MMboL55kCIBY7nvF0Hc5W/GXMLCef4eDQ/aZn\nuqgqsrcdMOsW0Ugv+iT7AoGAMhVEOuTLsub22VcPp0Xyy5RjEBlJ5LRzKy6HWe2q\nH6ubKQyQ07iGOuYTdQR9FgNTTY8ZjQRa3nC8Eve4/49rypL5gjkVh9NkAECqeZ+I\nkCOUyrS8ruV2a1XGahJA4/Os2myttcrxZ4Sn222lthYtpV3g6UhDgAHggJtV4tuU\ngO0CgYEAqCrkYsBaVBiCFzuTyyXjU2wYz303skO2iizcIViE5PQXeqwy6JHJuFZa\nMtuTaGKSaFHgRNdyQwlcTMlndrv7bIkhr6fif496Dadh5UqeX/60labOjX716EOk\n7JiUCeJdbXPoWLokTggzeLz1mD6AT/ZnvBEzfe8oT9UntkLXT5I=\n-----END RSA PRIVATE KEY-----')] - :param server_list: CMDB主机列表 - :return: - """ - obj = RsyncPublicKey() - return obj.sync_key(server_list) - - -if __name__ == '__main__': - fire.Fire(start_rsync) +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Time : 2019/5/1 13:28 +# @Author : Fred Yangxiaofei +# @File : sync_public_key.py +# @Role : 推送公钥到主机，实现免密钥登陆 + +import os +import paramiko +from settings import PUBLIC_KEY +from models.server import SSHConfigs +from libs.db_context import DBContext +from libs.web_logs import ins_log +from libs.common import remote_upload_file, get_key_file, exec_shell +import fire + +BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +class RsyncPublicKey(): + def __init__(self): + self.msg = '' + + def check_public_path(self): + """ + 检查是否有这个目录，没有目录就新建 + :return: + """ + PUBLIC_KEY_PATH = os.path.dirname(PUBLIC_KEY) + cmd = '[ ! -d {} ] && mkdir {} && chmod 700 {} ; '.format(PUBLIC_KEY_PATH, PUBLIC_KEY_PATH, PUBLIC_KEY_PATH) + code, ret = exec_shell(cmd) + if code == 0: + return True + else: + return False + + def init_rsa(self): + '''Server端生成秘钥对''' + cmd = 'ssh-keygen -t rsa -P "" -f {}/id_rsa'.format(os.path.dirname((PUBLIC_KEY))) + + code, ret = exec_shell(cmd) + if code == 0: + return True + else: + return False + + def save_of(self): + '''把读取或者生成的秘钥信息保存到SQL''' + with open('%s/id_rsa' % os.path.dirname(PUBLIC_KEY), 'r') as id_rsa, open( + '%s/id_rsa.pub' % os.path.dirname(PUBLIC_KEY), 'r') as id_rsa_pub: + with DBContext('w') as session: + new_config = SSHConfigs(name='cmdb', id_rsa=id_rsa.read(), id_rsa_pub=id_rsa_pub.read()) + session.add(new_config) + session.commit() + + def check_rsa(self): + """ + 检查CMDB 密钥配置,没有则创建新的写入数据库 + :return: + """ + # 检查路径 + self.check_public_path() + with DBContext('r') as session: + # 这张表里面直有一条信息，名字：cmdb， 一对密钥 + exist_rsa = session.query(SSHConfigs.id, SSHConfigs.id_rsa_pub, SSHConfigs.id_rsa).filter( + SSHConfigs.name == 'cmdb').first() + if not exist_rsa: + # 检查本地是否存在 + local_id_rsa_exist = os.path.exists('{}/id_rsa'.format(os.path.dirname(PUBLIC_KEY))) + if not local_id_rsa_exist: + check_rsa = self.init_rsa() + if not check_rsa: + return False + self.save_of() + + elif exist_rsa: + PUBLIC_KEY_PATH = os.path.dirname(PUBLIC_KEY) + id_rsa_pub = exist_rsa[1] + id_rsa = exist_rsa[2] + cmd1 = 'echo "{}" > {}/id_rsa.pub'.format(id_rsa_pub, PUBLIC_KEY_PATH) + cmd2 = 'echo "{}" > {}/id_rsa && chmod 600 {}/id_rsa'.format(id_rsa, PUBLIC_KEY_PATH, PUBLIC_KEY_PATH) + exec_shell(cmd1) + exec_shell(cmd2) + return True + else: + return True + + def sync_key(self, server_list): + """ + 批量下发server端公钥到client端 + :param server_list: 主机信息，IP端口用户密码 + :return: + """ + if not isinstance(server_list, list): + raise ValueError() + + ip = server_list[0][0] + port = server_list[0][1] + user = server_list[0][2] + user_key = server_list[0][3] + cmd = '[ ! -d ~/.ssh ] && mkdir ~/.ssh && chmod 700 ~/.ssh ; ' \ + '[ ! -f ~/.ssh/authorized_keys ] && touch ~/.ssh/authorized_keys; ' \ + 'grep -c "`cat /tmp/id_rsa.pub`" ~/.ssh/authorized_keys >> /dev/null;' \ + '[ $? == 0 ] || cat /tmp/id_rsa.pub >> ~/.ssh/authorized_keys && chmod 600 ~/.ssh/authorized_keys && echo ok' + # 将key写到本地 + ssh_key_file = get_key_file(user_key) + if ssh_key_file: + try: + res = remote_upload_file(ip, user, ssh_key_file, cmd, PUBLIC_KEY, '/tmp/id_rsa.pub', port) + if res[0] == 'ok': + self.msg = { + 'status': True, + 'ip': ip, + 'port': port, + 'user': user, + 'msg': '推送成功' + } + else: + # 状态改为False + self.msg = { + 'status': False, + 'ip': ip, + 'msg': '推送失败' + } + except paramiko.ssh_exception.AuthenticationException: + self.msg = { + 'status': False, + 'ip': ip, + 'msg': '认证失败，请检查管理用户Key是否正确' + } + + except Exception as e: + print(e) + self.msg = { + 'status': False, + 'ip': ip, + 'msg': '{}'.format(e) + } + + ins_log.read_log('info', self.msg) + return self.msg + + +def start_rsync(server_list): + """ + 推送CMDB公钥 + # server_list = [('172.16.0.120', 22, 'root', '-----BEGIN RSA PRIVATE KEY-----\nMIIEogIBAAKCAQEAlTbeMPz7G69rRtpAdlcR/YIKd17oFJBcQ6+2ELTs9tA5Tc/W\nZo3+ftex8pKXx9sGyH28ynrwBiJTRmyaSNFssIxDXGbS9mbliLF+3NZX8sE8soUZ\n1bJYhF3VmKhziRgcRr4QKMltVNMFI/p8ofdsrbNFX6f2OA0iP3bHTbTUzgDiRjwx\nubhXolog/eZboCTahKClPlT3ffhoyYihLX8zjgAxJsFpUAFfVXUrGjD+ttEPoSjg\nxhJRdfXMHF9QLmzMwyxJvvreKtXM+LzyrMRcdAedt3mF+BWc89XhqglM5IcR6fIm\nRyXf3dn7mgWm42xGILSOT6qSjJ1JyptkVc7/5wIBIwKCAQBMvSHQDRNtUua2uWL7\nCDUjSjFEtHdbCGn27KbGw3KNj6hxKQ8tbZjThN8sS2QAYgN9DXcJDAZpjf7xBKce\nIo+rQNIvhVaNWX1cPga6t2BuN35NLoJQpN0Q38zvbJMwkA6KyCWRUaXxHBFFtAWV\nIClDY4LDevOnzD1/uW3QMRz8Oi7/Awrce0QX42O8EkKXwf7YtV0ZIfRiKxaNKJVS\nWqxkVCxXdcAwpq6E4zaEXmBTISc2QhFD1n6yS9Z74rSAF2fIVErbBHAGRHMcUpdu\nR3ALKxo4/6R0KQ3rRj/SB8aMPp7KrR2ebSYWAlKDxete66MjIfBXQ1sr/the7iqV\nUJzLAoGBAMQgHreTPrfxtVj2hfpMl+GBvr/VyOoBVobJOZLi4/docuX2fpGVltGb\nb7Lq6F+Wd06nmmHqHhaBTgv2Vw9c8W8svhMarLs96EF/E53p7Xy3f2SmOOUoFbXO\nkMkRxPDK3uEOjDfvptiSuneaDpkbt7B9h+ipaDaTUunSYdgy4oRRAoGBAMLEex3r\nifCeo4YZrqbzk8JdgT6bO869v+DRY7MOB163zS3biIyxJnNF2eODO8diQ/S6uLKf\nKH19hhDfz3ARTdrmPBbd/mWLaHLClvVsaGlt7U0wpOEKk6FqA5x/3+hB2RiKMcL4\nxz08xaqqZ/0A6mal+OCKzZx5E7s/o1XHs0q3AoGBALjrFaXFWGRKS+YqQ869Pr6t\nh/azoC0XNFMywUFZlRx4bFUqPNJoeEH49Embp+UY1uPCmN/5/xyIi2pdSsVXoc84\n0Hhpm4v4ioblh4ZC9dv9eB0RwJ2MMboL55kCIBY7nvF0Hc5W/GXMLCef4eDQ/aZn\nuqgqsrcdMOsW0Ugv+iT7AoGAMhVEOuTLsub22VcPp0Xyy5RjEBlJ5LRzKy6HWe2q\nH6ubKQyQ07iGOuYTdQR9FgNTTY8ZjQRa3nC8Eve4/49rypL5gjkVh9NkAECqeZ+I\nkCOUyrS8ruV2a1XGahJA4/Os2myttcrxZ4Sn222lthYtpV3g6UhDgAHggJtV4tuU\ngO0CgYEAqCrkYsBaVBiCFzuTyyXjU2wYz303skO2iizcIViE5PQXeqwy6JHJuFZa\nMtuTaGKSaFHgRNdyQwlcTMlndrv7bIkhr6fif496Dadh5UqeX/60labOjX716EOk\n7JiUCeJdbXPoWLokTggzeLz1mD6AT/ZnvBEzfe8oT9UntkLXT5I=\n-----END RSA PRIVATE KEY-----')] + :param server_list: CMDB主机列表 + :return: + """ + obj = RsyncPublicKey() + return obj.sync_key(server_list) + + +if __name__ == '__main__': + fire.Fire(start_rsync) diff --git a/libs/server/sync_to_tagtree.py b/libs/server/sync_to_tagtree.py index 50fbcea..7e1cf1f 100644 --- a/libs/server/sync_to_tagtree.py +++ b/libs/server/sync_to_tagtree.py @@ -1,125 +1,125 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# @Time : 2019/5/21 10:35 -# @Author : Fred Yangxiaofei -# @File : sync_to_tagtree.py -# @Role : 将CMDB里面的数据同步到作业配置---标签树下面 - - - -from models.server import Server, model_to_dict -from libs.db_context import DBContext -from libs.web_logs import ins_log -from settings import CODO_TASK_DB_INFO -from opssdk.operate.mysql import MysqlBase -import fire - - -class SyncTagTree(): - def __init__(self, mb): - self.mb = mb - self.server_table = 'asset_server' # codo-task tagtree server表名字，默认无需更改 - self.server_tag_table = 'asset_server_tag' # codo-tasks'tag tree server和标签的关联表 - - def get_asset_server(self): - """ - 获取CMDB里面所有主机 - :return: - """ - server_list = [] - with DBContext('r') as session: - server_info = session.query(Server).order_by(Server.id).all() - for msg in server_info: - data_dict = model_to_dict(msg) - data_dict['create_time'] = str(data_dict['create_time']) - server_list.append(data_dict) - - return server_list - - def server_sync_task(self): - """ - server信息同步到codo-task里面的tag tree数据库里面 - :return: - """ - - server_list = self.get_asset_server() - if not server_list: - print('[Error]: 没有获取到Server信息') - return False - - cmdb_all_hostname_list = [] - for server in server_list: - hostname = server.get('hostname') - ip = server.get('ip') - idc = server.get('idc') - region = server.get('region') - state = server.get('state') - detail = server.get('detail') - cmdb_all_hostname_list.append(hostname) - - exist_hostname = "select * from `{table_name}` WHERE `hostname`= '{hostname}';".format( - table_name=self.server_table, - hostname=hostname) - # print(query_sql) - if self.mb.query(exist_hostname): - # print('主机名：{}是存在的,直接更新'.format(hostname)) - update_sql = "update `{table_name}` set `ip`='{ip}', `idc`='{idc}', `region`='{region}', `state`='{state}', `detail`='{detail}' where `hostname`='{hostname}';".format( - table_name=self.server_table, ip=ip, idc=idc, region=region, state=state, detail=detail, - hostname=hostname) - self.mb.change(update_sql) - # ins_log.read_log('info', '{}更新成功'.format(hostname)) - - else: - insert_sql = "insert into {table_name} (hostname,ip,idc,region,state,detail) VALUES ('{hostname}','{ip}','{idc}','{region}','{state}','{detail}');".format( - table_name=self.server_table, hostname=hostname, ip=ip, idc=idc, region=region, state=state, - detail=detail) - self.mb.change(insert_sql) - ins_log.read_log('info', '{}同步成功'.format(hostname)) - - # 如果CMDB数据库里面删除了主机，标签树也删除 主机和主机关联表 - query_sql = "select hostname, id from {table_name};".format(table_name=self.server_table) - - target_data = self.mb.query(query_sql) - for i in list(target_data): - if i[0] not in cmdb_all_hostname_list: - # print('我要删除这个name{}'.format(hostname[0])) - delete_hostname_sql = "DELETE FROM {table_name} WHERE `hostname`='{hostname}';".format( - table_name=self.server_table, hostname=i[0]) - delete_id_sql = "DELETE FROM {table_name} where `server_id`={server_id};".format( - table_name=self.server_tag_table, server_id=i[1]) - self.mb.change(delete_hostname_sql) - self.mb.change(delete_id_sql) - ins_log.read_log('info', '{}删除成功'.format(i[0])) # i[0]:hostname - ins_log.read_log('info', '{}删除成功'.format(i[1])) # i[1]: server_id - - -def main(): - """ - 检查用户是否配置了同步codo-task MySQL信息, - 如果检测到配置，初始化MySQL，同步数据 - 拿不到ORM直接使用原生SQL语句操作 - :return: - """ - host = CODO_TASK_DB_INFO.get('host') - port = CODO_TASK_DB_INFO.get('port') - user = CODO_TASK_DB_INFO.get('user') - passwd = CODO_TASK_DB_INFO.get('passwd') - db = CODO_TASK_DB_INFO.get('db') - - if not host or not port or not user or not passwd or not db: - print('[Error]: Not fount CODO_TASK_DB_INFO, auto pass...') - return False - - try: - mb = MysqlBase(**CODO_TASK_DB_INFO) - obj = SyncTagTree(mb) - obj.server_sync_task() - - except Exception as e: - msg = '[Error]: 请确认下CODO_TASK 数据库配置信息是否正确' - ins_log.read_log('error', e) - return msg - - -if __name__ == '__main__': - fire.Fire(main) +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Time : 2019/5/21 10:35 +# @Author : Fred Yangxiaofei +# @File : sync_to_tagtree.py +# @Role : 将CMDB里面的数据同步到作业配置---标签树下面 + + + +from models.server import Server, model_to_dict +from libs.db_context import DBContext +from libs.web_logs import ins_log +from settings import CODO_TASK_DB_INFO +from opssdk.operate.mysql import MysqlBase +import fire + + +class SyncTagTree(): + def __init__(self, mb): + self.mb = mb + self.server_table = 'asset_server' # codo-task tagtree server表名字，默认无需更改 + self.server_tag_table = 'asset_server_tag' # codo-tasks'tag tree server和标签的关联表 + + def get_asset_server(self): + """ + 获取CMDB里面所有主机 + :return: + """ + server_list = [] + with DBContext('r') as session: + server_info = session.query(Server).order_by(Server.id).all() + for msg in server_info: + data_dict = model_to_dict(msg) + data_dict['create_time'] = str(data_dict['create_time']) + server_list.append(data_dict) + + return server_list + + def server_sync_task(self): + """ + server信息同步到codo-task里面的tag tree数据库里面 + :return: + """ + + server_list = self.get_asset_server() + if not server_list: + print('[Error]: 没有获取到Server信息') + return False + + cmdb_all_hostname_list = [] + for server in server_list: + hostname = server.get('hostname') + ip = server.get('ip') + idc = server.get('idc') + region = server.get('region') + state = server.get('state') + detail = server.get('detail') + cmdb_all_hostname_list.append(hostname) + + exist_hostname = "select * from `{table_name}` WHERE `hostname`= '{hostname}';".format( + table_name=self.server_table, + hostname=hostname) + # print(query_sql) + if self.mb.query(exist_hostname): + # print('主机名：{}是存在的,直接更新'.format(hostname)) + update_sql = "update `{table_name}` set `ip`='{ip}', `idc`='{idc}', `region`='{region}', `state`='{state}', `detail`='{detail}' where `hostname`='{hostname}';".format( + table_name=self.server_table, ip=ip, idc=idc, region=region, state=state, detail=detail, + hostname=hostname) + self.mb.change(update_sql) + # ins_log.read_log('info', '{}更新成功'.format(hostname)) + + else: + insert_sql = "insert into {table_name} (hostname,ip,idc,region,state,detail) VALUES ('{hostname}','{ip}','{idc}','{region}','{state}','{detail}');".format( + table_name=self.server_table, hostname=hostname, ip=ip, idc=idc, region=region, state=state, + detail=detail) + self.mb.change(insert_sql) + ins_log.read_log('info', '{}同步成功'.format(hostname)) + + # 如果CMDB数据库里面删除了主机，标签树也删除 主机和主机关联表 + query_sql = "select hostname, id from {table_name};".format(table_name=self.server_table) + + target_data = self.mb.query(query_sql) + for i in list(target_data): + if i[0] not in cmdb_all_hostname_list: + # print('我要删除这个name{}'.format(hostname[0])) + delete_hostname_sql = "DELETE FROM {table_name} WHERE `hostname`='{hostname}';".format( + table_name=self.server_table, hostname=i[0]) + delete_id_sql = "DELETE FROM {table_name} where `server_id`={server_id};".format( + table_name=self.server_tag_table, server_id=i[1]) + self.mb.change(delete_hostname_sql) + self.mb.change(delete_id_sql) + ins_log.read_log('info', '{}删除成功'.format(i[0])) # i[0]:hostname + ins_log.read_log('info', '{}删除成功'.format(i[1])) # i[1]: server_id + + +def main(): + """ + 检查用户是否配置了同步codo-task MySQL信息, + 如果检测到配置，初始化MySQL，同步数据 + 拿不到ORM直接使用原生SQL语句操作 + :return: + """ + host = CODO_TASK_DB_INFO.get('host') + port = CODO_TASK_DB_INFO.get('port') + user = CODO_TASK_DB_INFO.get('user') + passwd = CODO_TASK_DB_INFO.get('passwd') + db = CODO_TASK_DB_INFO.get('db') + + if not host or not port or not user or not passwd or not db: + print('[Error]: Not fount CODO_TASK_DB_INFO, auto pass...') + return False + + try: + mb = MysqlBase(**CODO_TASK_DB_INFO) + obj = SyncTagTree(mb) + obj.server_sync_task() + + except Exception as e: + msg = '[Error]: 请确认下CODO_TASK 数据库配置信息是否正确' + ins_log.read_log('error', e) + return msg + + +if __name__ == '__main__': + fire.Fire(main) diff --git a/libs/web_logs.py b/libs/web_logs.py index 77562b1..fb25d40 100644 --- a/libs/web_logs.py +++ b/libs/web_logs.py @@ -1,125 +1,125 @@ -#!/usr/bin/env python -# -*-coding:utf-8-*- -''' -Author : ss -date : 2018-3-19 -role : web log -''' - -import logging -import os -import time -from shortuuid import uuid - -log_fmt = ''.join(('PROGRESS:%(progress_id) -5s %(levelname) ', '-10s %(asctime)s %(name) -25s %(funcName) ' - '-30s LINE.NO:%(lineno) -5d : %(message)s')) -log_key = 'logger_key' - - -def singleton(class_): - instances = {} - - def getinstance(*args, **kwargs): - if class_ not in instances: - instances[class_] = class_(*args, **kwargs) - return instances[class_] - - return getinstance - - -class ProgressLogFilter(logging.Filter): - def filter(self, record): - record.progress_id = Logger().progress_id - return True - - -@singleton -class Logger(object): - def __init__(self, progress_id='', log_file='/tmp/xxx.log'): - self.__log_key = log_key - self.progress_id = progress_id - self.log_file = log_file - - def read_log(self, log_level, log_message): - ###创建一个logger - if self.progress_id == '': - Logger().progress_id = str(uuid()) - else: - Logger().progress_id = self.progress_id - logger = logging.getLogger(self.__log_key) - logger.addFilter(ProgressLogFilter()) - logger.setLevel(logging.DEBUG) - - ###创建一个handler用于输出到终端 - th = logging.StreamHandler() - th.setLevel(logging.DEBUG) - - ###定义handler的输出格式 - formatter = logging.Formatter(log_fmt) - th.setFormatter(formatter) - - ###给logger添加handler - logger.addHandler(th) - - ###记录日志 - level_dic = {'debug': logger.debug, 'info': logger.info, 'warning': logger.warning, 'error': logger.error, - 'critical': logger.critical} - level_dic[log_level](log_message) - - th.flush() - logger.removeHandler(th) - - def write_log(self, log_level, log_message): - ###创建一个logger - ###创建一个logger - if self.progress_id == '': - Logger().progress_id = str(uuid()) - else: - Logger().progress_id = self.progress_id - logger = logging.getLogger(self.__log_key) - logger.addFilter(ProgressLogFilter()) - logger.setLevel(logging.DEBUG) - - ###建立日志目录 - log_dir = os.path.dirname(self.log_file) - if not os.path.isdir(log_dir): - os.makedirs(log_dir) - - ###创建一个handler用于写入日志文件 - fh = logging.FileHandler(self.log_file) - fh.setLevel(logging.DEBUG) - - ###定义handler的输出格式 - formatter = logging.Formatter(log_fmt) - fh.setFormatter(formatter) - - ###给logger添加handler - logger.addHandler(fh) - - ###记录日志 - level_dic = {'debug': logger.debug, 'info': logger.info, 'warning': logger.warning, 'error': logger.error, - 'critical': logger.critical} - level_dic[log_level](log_message) - - ###删除重复记录 - fh.flush() - logger.removeHandler(fh) - - -ins_log = Logger() - - -def timeit(func): - def wrapper(*args, **kwargs): - start_time = time.time() - result = func(*args, **kwargs) - end_time = time.time() - duration = end_time - start_time - ins_log.read_log('info', '%s execute duration :%.3f second' % (str(func), duration)) - return result - - return wrapper - - -# ins_log.write_log('info', 'xxxx') +#!/usr/bin/env python +# -*-coding:utf-8-*- +''' +Author : ss +date : 2018-3-19 +role : web log +''' + +import logging +import os +import time +from shortuuid import uuid + +log_fmt = ''.join(('PROGRESS:%(progress_id) -5s %(levelname) ', '-10s %(asctime)s %(name) -25s %(funcName) ' + '-30s LINE.NO:%(lineno) -5d : %(message)s')) +log_key = 'logger_key' + + +def singleton(class_): + instances = {} + + def getinstance(*args, **kwargs): + if class_ not in instances: + instances[class_] = class_(*args, **kwargs) + return instances[class_] + + return getinstance + + +class ProgressLogFilter(logging.Filter): + def filter(self, record): + record.progress_id = Logger().progress_id + return True + + +@singleton +class Logger(object): + def __init__(self, progress_id='', log_file='/tmp/xxx.log'): + self.__log_key = log_key + self.progress_id = progress_id + self.log_file = log_file + + def read_log(self, log_level, log_message): + ###创建一个logger + if self.progress_id == '': + Logger().progress_id = str(uuid()) + else: + Logger().progress_id = self.progress_id + logger = logging.getLogger(self.__log_key) + logger.addFilter(ProgressLogFilter()) + logger.setLevel(logging.DEBUG) + + ###创建一个handler用于输出到终端 + th = logging.StreamHandler() + th.setLevel(logging.DEBUG) + + ###定义handler的输出格式 + formatter = logging.Formatter(log_fmt) + th.setFormatter(formatter) + + ###给logger添加handler + logger.addHandler(th) + + ###记录日志 + level_dic = {'debug': logger.debug, 'info': logger.info, 'warning': logger.warning, 'error': logger.error, + 'critical': logger.critical} + level_dic[log_level](log_message) + + th.flush() + logger.removeHandler(th) + + def write_log(self, log_level, log_message): + ###创建一个logger + ###创建一个logger + if self.progress_id == '': + Logger().progress_id = str(uuid()) + else: + Logger().progress_id = self.progress_id + logger = logging.getLogger(self.__log_key) + logger.addFilter(ProgressLogFilter()) + logger.setLevel(logging.DEBUG) + + ###建立日志目录 + log_dir = os.path.dirname(self.log_file) + if not os.path.isdir(log_dir): + os.makedirs(log_dir) + + ###创建一个handler用于写入日志文件 + fh = logging.FileHandler(self.log_file) + fh.setLevel(logging.DEBUG) + + ###定义handler的输出格式 + formatter = logging.Formatter(log_fmt) + fh.setFormatter(formatter) + + ###给logger添加handler + logger.addHandler(fh) + + ###记录日志 + level_dic = {'debug': logger.debug, 'info': logger.info, 'warning': logger.warning, 'error': logger.error, + 'critical': logger.critical} + level_dic[log_level](log_message) + + ###删除重复记录 + fh.flush() + logger.removeHandler(fh) + + +ins_log = Logger() + + +def timeit(func): + def wrapper(*args, **kwargs): + start_time = time.time() + result = func(*args, **kwargs) + end_time = time.time() + duration = end_time - start_time + ins_log.read_log('info', '%s execute duration :%.3f second' % (str(func), duration)) + return result + + return wrapper + + +# ins_log.write_log('info', 'xxxx') #ins_log.read_log('info', 'xxxx') \ No newline at end of file diff --git a/models/db.py b/models/db.py index 235f1d1..dafa6bf 100644 --- a/models/db.py +++ b/models/db.py @@ -1,67 +1,67 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# @Time : 2019/4/17 13:48 -# @Author : Fred Yangxiaofei -# @File : db.py -# @Role : ORM - - -from sqlalchemy import Column, String, Integer, Text, DateTime -from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy.orm import class_mapper -from datetime import datetime - -Base = declarative_base() - - -def model_to_dict(model): - model_dict = {} - for key, column in class_mapper(model.__class__).c.items(): - model_dict[column.name] = getattr(model, key, None) - return model_dict - - -class DBTag(Base): - __tablename__ = 'asset_db_tag' - - id = Column(Integer, primary_key=True, autoincrement=True) - db_id = Column('db_id', Integer) - tag_id = Column('tag_id', Integer) - - -class DB(Base): - __tablename__ = 'asset_db' - ### 数据库集群 - id = Column(Integer, primary_key=True, autoincrement=True) - idc = Column('idc', String(128)) # IDC - db_instance_id = Column('db_instance_id', String(128)) #RDS实例ID - db_code = Column('db_code', String(255)) ### 名称 代号 编码 - db_class = Column('db_class', String(255)) ### DB实例类型 - db_host = Column('db_host', String(255), nullable=False) ### DB主机地址 - db_public_ip = Column('db_public_ip', String(255)) ### DB主机外网地址,只有少量的才会开启 - db_port = Column('db_port', String(10), nullable=False, default=3306) ### DB端口 - db_user = Column('db_user', String(128), nullable=False, default='root') ### DB用户 - db_pwd = Column('db_pwd', String(128)) ### DB的密码 - db_disk = Column('db_disk', String(128)) ### DB的磁盘，主要RDS有 - db_region = Column('db_region', String(128)) ### DB的区域 可用区 - db_env = Column('db_env', String(128), default='release') ### 环境/release/dev - db_type = Column('db_type', String(128)) ### 标记类型如：MySQL/Redis - db_version = Column('db_version', String(128)) ###DB版本 - db_mark = Column('db_mark', String(255)) ### 标记读写备 - state = Column('state', String(128)) ### DB的状态 - db_detail = Column('db_detail', String(255)) ### 描述 - proxy_host = Column('proxy_host', String(128)) ### 代理主机 适配多云,预留 - create_time = Column('create_time', DateTime(), default=datetime.now) # 创建时间 - update_time = Column('update_time', DateTime(), default=datetime.now, onupdate=datetime.now) # 记录更新时间 - - - -# class ProxyInfo(Base): -# __tablename__ = 'asset_proxy_info' -# -# ### 代理主机 通过此主机来连接数据库 -# id = Column('id', Integer, primary_key=True, autoincrement=True) -# proxy_host = Column('proxy_host', String(60), unique=True, nullable=False) -# inception = Column('inception', String(300)) -# salt = Column('salt', String(300)) -# detail = Column('detail', String(20)) +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Time : 2019/4/17 13:48 +# @Author : Fred Yangxiaofei +# @File : db.py +# @Role : ORM + + +from sqlalchemy import Column, String, Integer, Text, DateTime +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import class_mapper +from datetime import datetime + +Base = declarative_base() + + +def model_to_dict(model): + model_dict = {} + for key, column in class_mapper(model.__class__).c.items(): + model_dict[column.name] = getattr(model, key, None) + return model_dict + + +class DBTag(Base): + __tablename__ = 'asset_db_tag' + + id = Column(Integer, primary_key=True, autoincrement=True) + db_id = Column('db_id', Integer) + tag_id = Column('tag_id', Integer) + + +class DB(Base): + __tablename__ = 'asset_db' + ### 数据库集群 + id = Column(Integer, primary_key=True, autoincrement=True) + idc = Column('idc', String(128)) # IDC + db_instance_id = Column('db_instance_id', String(128)) #RDS实例ID + db_code = Column('db_code', String(255)) ### 名称 代号 编码 + db_class = Column('db_class', String(255)) ### DB实例类型 + db_host = Column('db_host', String(255), nullable=False) ### DB主机地址 + db_public_ip = Column('db_public_ip', String(255)) ### DB主机外网地址,只有少量的才会开启 + db_port = Column('db_port', String(10), nullable=False, default=3306) ### DB端口 + db_user = Column('db_user', String(128), nullable=False, default='root') ### DB用户 + db_pwd = Column('db_pwd', String(128)) ### DB的密码 + db_disk = Column('db_disk', String(128)) ### DB的磁盘，主要RDS有 + db_region = Column('db_region', String(128)) ### DB的区域 可用区 + db_env = Column('db_env', String(128), default='release') ### 环境/release/dev + db_type = Column('db_type', String(128)) ### 标记类型如：MySQL/Redis + db_version = Column('db_version', String(128)) ###DB版本 + db_mark = Column('db_mark', String(255)) ### 标记读写备 + state = Column('state', String(128)) ### DB的状态 + db_detail = Column('db_detail', String(255)) ### 描述 + proxy_host = Column('proxy_host', String(128)) ### 代理主机 适配多云,预留 + create_time = Column('create_time', DateTime(), default=datetime.now) # 创建时间 + update_time = Column('update_time', DateTime(), default=datetime.now, onupdate=datetime.now) # 记录更新时间 + + + +# class ProxyInfo(Base): +# __tablename__ = 'asset_proxy_info' +# +# ### 代理主机 通过此主机来连接数据库 +# id = Column('id', Integer, primary_key=True, autoincrement=True) +# proxy_host = Column('proxy_host', String(60), unique=True, nullable=False) +# inception = Column('inception', String(300)) +# salt = Column('salt', String(300)) +# detail = Column('detail', String(20)) diff --git a/doc/requirements.txt b/requirements.txt similarity index 100% rename from doc/requirements.txt rename to requirements.txt diff --git a/startup.py b/startup.py index 785a097..bdadc86 100644 --- a/startup.py +++ b/startup.py @@ -1,42 +1,42 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# @Time : 2019/4/17 9:52 -# @Author : Fred Yangxiaofei -# @File : startup.py -# @Role : 启动脚本 - - -import fire -from tornado.options import define -from websdk.program import MainProgram -from settings import settings as app_settings -from biz.applications import Application as CmdbApp -from biz.crontab_app import Application as CronApp - -define("service", default='api', help="start service flag", type=str) - - -class MyProgram(MainProgram): - def __init__(self, service='cmdb_api', progressid=''): - self.__app = None - settings = app_settings - if service == 'cmdb': - self.__app = CmdbApp(**settings) - elif service == 'cmdb_cron': - self.__app = CronApp(**settings) - super(MyProgram, self).__init__(progressid) - self.__app.start_server() - - -if __name__ == '__main__': - fire.Fire(MyProgram) - -""" -#master -python3 startup.py --service='cmdb' --port=8055 - -#crontab -python3 startup.py --service='cmdb_cron' - - -""" +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Time : 2019/4/17 9:52 +# @Author : Fred Yangxiaofei +# @File : startup.py +# @Role : 启动脚本 + + +import fire +from tornado.options import define +from websdk.program import MainProgram +from settings import settings as app_settings +from biz.applications import Application as CmdbApp +from biz.crontab_app import Application as CronApp + +define("service", default='api', help="start service flag", type=str) + + +class MyProgram(MainProgram): + def __init__(self, service='cmdb_api', progressid=''): + self.__app = None + settings = app_settings + if service == 'cmdb': + self.__app = CmdbApp(**settings) + elif service == 'cmdb_cron': + self.__app = CronApp(**settings) + super(MyProgram, self).__init__(progressid) + self.__app.start_server() + + +if __name__ == '__main__': + fire.Fire(MyProgram) + +""" +#master +python3 startup.py --service='cmdb' --port=8055 + +#crontab +python3 startup.py --service='cmdb_cron' + + +""" diff --git a/static/__init__.py b/static/__init__.py index fd40d8f..e2086e2 100644 --- a/static/__init__.py +++ b/static/__init__.py @@ -1,6 +1,6 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# @Time : 2019/5/23 13:09 -# @Author : Fred Yangxiaofei -# @File : __init__.py.py +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Time : 2019/5/23 13:09 +# @Author : Fred Yangxiaofei +# @File : __init__.py.py # @Role : 说明脚本功能 \ No newline at end of file diff --git a/static/images/__init__.py b/static/images/__init__.py index fd40d8f..e2086e2 100644 --- a/static/images/__init__.py +++ b/static/images/__init__.py @@ -1,6 +1,6 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# @Time : 2019/5/23 13:09 -# @Author : Fred Yangxiaofei -# @File : __init__.py.py +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Time : 2019/5/23 13:09 +# @Author : Fred Yangxiaofei +# @File : __init__.py.py # @Role : 说明脚本功能 \ No newline at end of file