Python快速实现一个域名、IP信息聚合网站

发表于 | 更新于: | 分类于 ， ， | | 阅读次数:
字数统计: 2.3k | 阅读时长 ≈ 11

域名和IP地址信息是非常基础的情报信息，目前网上有很多网站都提供了域名信息的查询、IP地址及归属地的查询。本文通过Python Flask实现域名及IP情报信息的聚合网站。

因为域名和IP地址信息会有变化，为了减少接口压力，做了本地数据库的存储，新鲜度保存一周，每次查询先从本地数据库获取信息，如果本地库信息有并且没有超过一个星期就从本地库取，没有就从其他网站获取，并更新到本地库。

一、获取域名WHOIS信息

网上提供域名WHOIS信息查询的网站有很多，这里以http://whois.chinafu.com 为例实现WHOIS信息的查询和解析。

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
import requests
from bs4 import BeautifulSoup
headers = {
    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36'
}

def getwhoisinfobychinafu(domain):
    ret_result = {}
    result=getWhoisInfoFromDB(domain)
    if len(result)==0:
        whois_service_url = 'http://whois.chinafu.com/whois.php'
        post_data={"domain":domain}

        try:
            post_result=requests.post(whois_service_url,post_data)
            if post_result.status_code == 200:
                ret_str = post_result.content.decode('utf-8')
                soup = BeautifulSoup(ret_str, 'lxml')
                items_tr =soup.find(name='table',attrs={'class':'listtable'}).find_all(name='tr')
                for item_tr in items_tr:
                    td_item=item_tr.find(name='td')
                    if 'colspan' in td_item.attrs:
                        key_name='详情'
                        key_value=td_item.find(name='div',id='tab1').text
                    else:
                        key_name=item_tr.find(name='th').text
                        key_value=item_tr.find(name='td').text
                    ret_result[key_name]=key_value
                addchinafuWhoisInfo2DB(ret_result)
        except Exception as r:
            print('未知错误 %s' % (r))
    #ret_result = json.dumps(ret_result, ensure_ascii=False)
    else:
        ret_result=result[0]
    return ret_result

def getWhoisInfoFromDB(domainname):
    whoisInfos=db.session.execute('select * from whoisinfo where domain_name="%s" and updated_time > DATE_SUB(CURDATE(), INTERVAL 1 WEEK)' % domainname).fetchall()
    whoisInfo_dics=[]
    for whoisInfo in whoisInfos:
        chinafuwhoisinfo_dic=chinafuwhoisinfo2dic(whoisInfo)
        whoisInfo_dics.append(chinafuwhoisinfo_dic)
    return whoisInfo_dics

def addchinafuWhoisInfo2DB(chinafuWhoisInfo_dic):
    chinafuWhois=WhoisInfo()
    chinafuWhois.domain_name=chinafuWhoisInfo_dic.get('域名DomainName')
    chinafuWhois.domain_status=chinafuWhoisInfo_dic.get('域名状态Domain Status','')
    chinafuWhois.registrar=chinafuWhoisInfo_dic.get('注册商Sponsoring Registrar','')
    chinafuWhois.name_server=chinafuWhoisInfo_dic.get('DNS 服务器Name Server','')
    chinafuWhois.registrar_creation_date=chinafuWhoisInfo_dic.get('注册日期Registration Date','')
    chinafuWhois.registrar_updated_date = chinafuWhoisInfo_dic.get('更新日期Update Date', '')
    chinafuWhois.registrar_expiry_date = chinafuWhoisInfo_dic.get('到期日期Expiration Date', '')
    chinafuWhois.detail=chinafuWhoisInfo_dic.get('详情', '')[0:10000]
    chinafuWhois.source = '中国福网'
    db.session.execute('delete from whoisinfo where domain_name="%s" and source="%s"' % (chinafuWhoisInfo_dic.get('域名DomainName'), chinafuWhois.source))
    db.session.add(chinafuWhois)
    db.session.commit()

这里为了减少直接从其他网站获取WHOIS信息的压力，做了本地数据库的存储，每次先从本地数据库取WHOIS的信息，如果本地库信息有并且没有超过一个星期就从本地库取，没有就从其他网站获取，并更新到本地库。这里getWhoisInfoFromDB实现了取新鲜度为1周的数据，addchinafuWhoisInfo2DB实现将获取的信息保存到本地数据库。

二、根据域名解析出IP

根据域名解析出IP代码：

1
2
3
4
5
6
7
8
def getIPbyDomain(domain):
    addr=''
    try:
        myaddr = socket.getaddrinfo(domain, 'http')
        addr=myaddr[0][4][0]
    except Exception as e:
        print(e)
    return addr

三、获取IP信息

获取IP信息的API接口也有很多，有淘宝的 https://ip.taobao.com/outGetIpInfo 、IPINFO http://ipinfo.io/、IPAPI http://ip-api.com/ 以及GeoLite2离线库等。

从淘宝IP获取IP信息

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
def getipinfobytaobao(ip):
    taobaoIp_url = 'https://ip.taobao.com/outGetIpInfo'
    post_data={"ip":ip,"accessKey":"alibaba-inc"}
    ret_ipinfo= {}
    try:
        return_data=requests.post(taobaoIp_url,post_data)
        #其中返回数据中code的值的含义为，0：成功，1：服务器异常，2：请求参数异常，3：服务器繁忙，4：个人qps超出
        return_json=json.loads(return_data.text)
        if return_json['code']==0:
            ret_ipinfo['ip']=return_json['data']['ip']
            ret_ipinfo['country']=return_json['data']['country']
            ret_ipinfo['region']=return_json['data']['region']
            ret_ipinfo['org']=''
            ret_ipinfo['city'] = return_json['data']['city']
            ret_ipinfo['isp']=return_json['data']['isp']
            ret_ipinfo['loc'] = ''
            ret_ipinfo['timezone'] = ''
            ret_ipinfo['source']='淘宝IP'
            addIPInfo2DB(ret_ipinfo)
    except Exception as e:
        print('未知错误 %s' % (e))
    return ret_ipinfo

从ipinfo.io获取IP信息

1
2
3
4
5
6
7
8
9
10
11
12
def getipinfobyipinfo(ip):
    api_url='http://ipinfo.io/'+ip
    ipinfo = {}
    try:
        req_return = requests.get(api_url)
        if req_return.status_code == 200:
            ipinfo = json.loads(req_return.text)
            ipinfo['source']='ipinfo.io'
            addIPInfo2DB(ipinfo)
    except Exception as e:
        print('未知错误 %s' % (e))
    return ipinfo

从ip-api.com获取IP信息

1
2
3
4
5
6
7
8
9
10
11
12
13
14
def getipinfobyipapi(ip):
    api_url='http://ip-api.com/json/'+ip
    ipinfo={}
    try:
        req_return=requests.get(api_url)
        if req_return.status_code==200:
            ipinfo=json.loads(req_return.text)
            ipinfo['ip'] = ip
            ipinfo['source'] = 'ip-api.com'
            ipinfo['loc'] = str(ipinfo['lat'])+','+str(ipinfo['lon'])
            addIPInfo2DB(ipinfo)
    except Exception as e:
        print('未知错误 %s' % (e))
    return ipinfo

从GeoLite离线库获取IP信息

如何获取GeoLite离线库及如何读取，详见：http://xiejava.ishareread.com/posts/2c5697c0/

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
def getipinfobygeoip2(ip):
    ipinfo={}
    dbdir=Config.geoLiteDBdir
    with geoip2.database.Reader(dbdir) as reader:
        response = reader.city(ip)
        ipinfo['ip'] =ip
        ipinfo['country'] = response.country.names['zh-CN']
        ipinfo['region'] =''
        ipinfo['city']=response.city.name
        ipinfo['org'] =''
        ipinfo['loc'] = str(response.location.latitude)+','+str(response.location.longitude)
        ipinfo['timezone'] = response.location.time_zone
        ipinfo['source'] = 'GeoIP'
        addIPInfo2DB(ipinfo)
    return ipinfo

四、搭建一个FLASK Web应用来查询聚合的域名、IP情报信息

1、FLASK Web应用的工程组织

工程组织

2、配置数据及读取配置数据

1）配置数据

配置数据分别放在.env及.flaskenv中，其中.env放的是工程中用到的数据库链接等比较私密的配置信息。.flaskenv放的是Flask运行环境的信息
.env的配置信息参考如下：

1
2
3
4
5
6
DEV_DATABASE_URI = 'mysql+pymysql://dbuser:yourpassword@127.0.0.1:3306/infocol_db_dev?charset=utf8'
TEST_DATABASE_URI = 'mysql+pymysql://dbuser:yourpassword@127.0.0.1:3306/infocol_db_test?charset=utf8'
PROD_DATABASE_URI = 'mysql+pymysql://dbuser:yourpassword@127.0.0.1:3306/infocol_db?charset=utf8'

SQLALCHEMY_TRACK_MODIFICATIONS = True
SECRET_KEY=your secret key

.falskenv配置信息参考如下：

1
FLASK_ENV=development

2）实现读取配置数据

通过config.py实现配置数据的读取及管理

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
import os
from dotenv import load_dotenv
basedir=os.path.abspath(os.path.dirname(__file__))
flaskenv_path=os.path.join(basedir,'.flaskenv')
env_path=os.path.join(basedir,'.env')
if os.path.exists(flaskenv_path):
    load_dotenv(flaskenv_path)
if os.path.exists(env_path):
    load_dotenv(env_path)

class Config:
    geoLiteDBdir=os.path.join(basedir,'GeoLite2\GeoLite2-City.mmdb')
    flaskenv = os.getenv('FLASK_ENV','development')
    SECRET_KEY=os.getenv('SECRET_KEY','123!@#')
    SQLALCHEMY_TRACK_MODIFICATIONS=os.getenv('SQLALCHEMY_TRACK_MODIFICATIONS')
    SQLALCHEMY_DATABASE_URI = os.getenv('DEV_DATABASE_URI')
    @staticmethod
    def init_app(app):
        pass

class DevelopmentConfig(Config):
    DEBUG=True
    SQLALCHEMY_DATABASE_URI = os.getenv('DEV_DATABASE_URI')

class TestingConfig(Config):
    TESTING=True
    SQLALCHEMY_DATABASE_URI = os.getenv('TEST_DATABASE_URI')

class ProductionConfig(Config):
    SQLALCHEMY_DATABASE_URI = os.getenv('PROD_DATABASE_URI')

config={
    'development':DevelopmentConfig,
    'testing':TestingConfig,
    'production':ProductionConfig,
    'default':DevelopmentConfig
}

3、界面及路由

界面很简单就一个域名/IP的输入框，输入域名或IP后去查询相应的域名信息或IP信息显示到界面上。
index界面

界面用jinjia2的模板
index.html代码如下：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
{% extends "bootstrap/base.html" %}
{% block title %}InfoCol{% endblock %}
{% block head %}
    {{ super() }}
    <style></style>
{% endblock %}
{% block body %}
    {% block navbar %}
    <div class="navbar navbar-inverse" role="navigation">
        <div class="container">
            <div class="navbar-header">
                <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse">
                    <span class="sr-only">Toggle navigation</span>
                    <span class="icon-bar"></span>
                    <span class="icon-bar"></span>
                    <span class="icon-bar"></span>
                </button>
                <a class="navbar-brand" href="/">InfoCol</a>
            </div>
            <div class="navbar-collapse collapse">
                <ul class="nav navbar-nav">
                    <li><a href="/">Home</a></li>
                </ul>
            </div>
        </div>
    </div>
    {% endblock %}
    {% block content %}
        <div class="container">
            <div class="page-header " >
                <form method="post" class="center-block">
                <div class="center-block" style="text-align:center">
                {{ form.hidden_tag() }}
                {{ form.name.label }}{{ form.name() }}
                {{ form.submit() }}
                </div>
                </form>
            </div>
            <div>
                {% if whois_info %}
                  <table class="table table-bordered">
                    <tr><th colspan="2">{{ name }}的Whois信息</th></tr>
                    {% for item in whois_info %}
                        {% if item!='详情' %}
                        <tr><td style="width: 20%">{{ item }}</td><td style="width: 80%">{{ whois_info[item] }}</td></tr>
                        {% else %}
                        <tr>
                            <td style="width: 20%">
                            <a role="button" data-toggle="collapse" href="#collapseExample" aria-expanded="false" aria-controls="collapseExample">
                               {{ item }}
                            </a>
                            </td>
                            <td style="width: 80%">
                                <div class="collapse" id="collapseExample">
                                  <div class="well">
                                    {{ whois_info[item] }}
                                  </div>
                                </div>
                            </td>
                        </tr>
                        {% endif %}
                    {% endfor %}
                  </table>
                {% endif %}
                {% if ipinfos|length>0 %}
                  <table class="table table-bordered">
                      <tr><th>IP</th><th>国家/地区</th><th>省份</th><th>城市</th><th>机构</th><th>ISP</th><th>经纬度</th><th>来源</th></tr>
                      {% for ipinfo in ipinfos %}
                      <tr>
                          <td>{{ ipinfo['ip'] }}</td>
                          <td>{{ ipinfo['country'] }}</td>
                          <td>{{ ipinfo['region'] }}</td>
                          <td>{{ ipinfo['city'] }}</td>
                          <td>{{ ipinfo['org'] }}</td>
                          <td>{{ ipinfo['isp'] }}</td>
                          <td>{{ ipinfo['loc'] }}</td>
                          <td>
                              {% if ipinfo['source']=='ipinfo.io' %}
                                  <a href="http://ipinfo.io/{{ ipinfo['ip'] }}" target="_blank">{{ ipinfo['source'] }}</a>
                              {% elif ipinfo['source']=='ip-api.com'%}
                                  <a href="http://ip-api.com/json/{{ ipinfo['ip'] }}" target="_blank">{{ ipinfo['source'] }}</a>
                              {% else %}
                                  {{ ipinfo['source'] }}
                              {% endif %}
                          </td>
                      </tr>
                      {% endfor %}
                  </table>
                {% endif %}
            </div>
        </div>
    {% endblock %}
{% endblock %}

路由配置处理中实现了获取表单中的信息，并判断是域名还是IP如果是域名者获取whois信息，并根据域名获取IP信息。如果输入的是IP则获取IP信息，并反馈到页面上。

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
@index_bp.route('/',methods=['GET','POST'])
def index():
    name = ''
    ipinfos = []
    whois_info = ''
    form = InputForm()
    if form.validate_on_submit():
        name = form.name.data
        if checkip(name):
            ipinfos = getipinfo(name)
        else:
            whois_info = getwhoisinfo(name)
            whois_ip = getIPbyDomain(name)
            if checkip(whois_ip):
                ipinfos = getipinfo(whois_ip)
        form.name.data = ''
    return render_template('index.html',form=form, name=name, ipinfos=ipinfos, whois_info=whois_info)

4、最终实现效果

界面查询效果

至此通过Python快速实现了一个简单的域名、IP信息聚合网站

全部源代码：https://github.com/xiejava1018/infocollect

演示地址：http://test.ishareread.com/

博客：http://xiejava.ishareread.com/


“fullbug”微信公众号

关注：微信公众号,一起学习成长！

0%