Skip to content

Commit a9f8993

Browse files
JasonJeJasonJe
committed
离线瓦片地图、Flask 实现 token
0 parents  commit a9f8993

16 files changed

+818
-0
lines changed

‎README.md

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
# :memo: `Memos!`
2+
3+
## [离线瓦片地图](./offline_tileserver)
4+
5+
## [`Flask`实现`api token`](./redis_token)

‎offline_tileserver/README.md

+88
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
# 离线瓦片地图
2+
3+
## 1. 获取瓦片地图
4+
5+
### 1.1 主要地图瓦片坐标系定义及计算原理
6+
7+
瓦片地图具有如下特点：
8+
9+
* 具有唯一的瓦片等级和瓦片坐标编号；
10+
11+
* 分辨率为 256*256；
12+
13+
* 瓦片等级越高，展示的地图信息约详尽，瓦片数量也越多；
14+
15+
* 上一等级的一张瓦片地图可分割为下一等级的四张瓦片地图。
16+
17+
参考文章： [《国内主要地图瓦片坐标系定义及计算原理》](https://blog.csdn.net/wudiazu/article/details/76597294)
18+
19+
### 1.2 [pyGeoTile 库](https://github.com/geometalab/pyGeoTile)的使用（计算瓦片地图坐标）
20+
21+
GitHub: https://github.com/geometalab/pyGeoTile
22+
23+
安装：`pip install pygeotile`
24+
25+
```
26+
from pygeotile.tile import Tile
27+
28+
lon, lat = 109.227, - 20.196
29+
zoom = 12
30+
31+
tile = Tile.for_latitude_longitude(latitude = lat, longitude = lon, zoom = zoom)
32+
print('mintile', 'X:', tile.tms_x, 'Y:', tile.tms_y, 'zoom:', tile.zoom)
33+
```
34+
35+
36+
### 1.3 多进程/多线程/异步爬取瓦片地图
37+
38+
* 高德瓦片地图链接：http://wprd03.is.autonavi.com/appmaptile?style=7&x={x}&y={y}&z={z}
39+
40+
* 谷歌瓦片地图链接：http://mt2.google.cn/vt/lyrs=m@167000000&hl=zh-CN&gl=cn&x={x}&y={y}&z={z}&s=Galil
41+
42+
三种方式爬取等级12瓦片地图（6164张图片）的效率对比
43+
44+
爬取方式：使用时间(秒)
45+
46+
* 多进程：24.49594s
47+
48+
* 多线程：16.11365s
49+
50+
* 异步：14.01809s
51+
52+
53+
## 2. Flask + Leaflet 瓦片地图
54+
55+
56+
### 2.1 瓦片地图路由定义
57+
58+
```
59+
@app.route("/tile")
60+
def tile():
61+
x = request.args['x']
62+
y = request.args['y']
63+
z = request.args['z']
64+
with open('./tilefile/%s/%s_%s.png'%(z, x, y), 'rb') as f:
65+
image = f.read()
66+
return Response(image, mimetype='image/jpeg')
67+
```
68+
69+
* 瓦片地图接口 url: /tile?x={x}&y={y}&z={z}
70+
71+
72+
### 2.2 [Leaflet 地图库](https://leafletjs.com/) 使用
73+
74+
GitHub：https://github.com/Leaflet/Leaflet
75+
76+
```
77+
var url = '/tile?x={x}&y={y}&z={z}';
78+
79+
80+
var latlng = new L.latLng(23.461, 111.921);
81+
var map = new L.map('mapDiv', {
82+
center: latlng,
83+
zoom: 4,
84+
detectRetina: true
85+
});
86+
87+
L.tileLayer(url).addTo(map);
88+
```

‎offline_tileserver/main.py

+27
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
import requests
2+
from flask import Flask, Response, jsonify, render_template, request
3+
from collections import namedtuple, Counter
4+
import json
5+
6+
Tile = namedtuple('Tile', ['x', 'y', 'z'])
7+
tile_counter = Counter()
8+
9+
app = Flask(__name__)
10+
11+
@app.route("/tile")
12+
def tile():
13+
x = request.args['x']
14+
y = request.args['y']
15+
z = request.args['z']
16+
t = Tile(x, y, z)
17+
with open('./tilefile/%s/%s_%s.png'%(z, x, y), 'rb') as f:
18+
image = f.read()
19+
tile_counter[t] += 1
20+
return Response(image, mimetype='image/jpeg')
21+
22+
@app.route('/', methods=['GET'])
23+
def map():
24+
return render_template('index.html')
25+
26+
if __name__ == "__main__":
27+
app.run(host='0.0.0.0', debug = True, threaded=True)

‎offline_tileserver/multiasync_tilemap.py

+110
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
## 异步爬取切片地图
2+
from pygeotile.tile import Tile
3+
from urllib import request
4+
import os
5+
import asyncio
6+
import time
7+
from time import sleep
8+
from asyncio import Queue
9+
import functools
10+
11+
def create_image_path(rootpath, level):
12+
path = './%s/%d'%(rootpath, level)
13+
if not os.path.exists(path):
14+
os.makedirs(path)
15+
return path
16+
17+
def create_image_url(minlon, minlat, maxlon, maxlat, minzoom, maxzoom, basetileurl, rootpath):
18+
for zoom in range(minzoom, maxzoom + 1):
19+
mintile = Tile.for_latitude_longitude(latitude = minlat, longitude = minlon, zoom = zoom)
20+
maxtile = Tile.for_latitude_longitude(latitude = maxlat, longitude = maxlon, zoom = zoom)
21+
22+
print('mintile', 'X:', mintile.tms_x, 'Y:', mintile.tms_y, 'zoom:', mintile.zoom)
23+
print('maxtile', 'Y:', maxtile.tms_x, 'Y:', maxtile.tms_y, 'zoom:', maxtile.zoom)
24+
25+
mintms_x, mintms_y = mintile.tms_x, mintile.tms_y
26+
maxtms_x, maxtms_y = maxtile.tms_x, maxtile.tms_y
27+
28+
create_image_path(rootpath, zoom)
29+
30+
imagelists = Queue()
31+
for x in range(mintms_x, maxtms_x + 1):
32+
for y in range(maxtms_y, mintms_y + 1):
33+
savepath = './%s/%d/%d_%d.png'%(rootpath, zoom, x, y)
34+
tileurl = basetileurl + '&x=%d&y=%d&z=%d'%(x, y, zoom)
35+
imagelists.put_nowait((tileurl, savepath))
36+
37+
return imagelists
38+
39+
async def save_image(imagelists):
40+
while True:
41+
try:
42+
start = time.time()
43+
print('---- Length', imagelists.qsize())
44+
if imagelists.empty():
45+
print("Done!")
46+
break
47+
image = await imagelists.get()
48+
tileurl, savepath = image[0], image[1]
49+
50+
request.urlretrieve(tileurl, savepath)
51+
print('---- PID:', os.getpid(), '--- use time: ', str(time.time() - start), tileurl)
52+
53+
imagelists.task_done()
54+
except Exception as e:
55+
print('---- Error: {}'.format(e))
56+
with open('./error.log', 'a') as f:
57+
f.write('---- Error: {}'.format(e))
58+
59+
def create_image_url_v2(minlon, minlat, maxlon, maxlat, minzoom, maxzoom, basetileurl, rootpath):
60+
for zoom in range(minzoom, maxzoom + 1):
61+
mintile = Tile.for_latitude_longitude(latitude = minlat, longitude = minlon, zoom = zoom)
62+
maxtile = Tile.for_latitude_longitude(latitude = maxlat, longitude = maxlon, zoom = zoom)
63+
64+
print('mintile', 'X:', mintile.tms_x, 'Y:', mintile.tms_y, 'zoom:', mintile.zoom)
65+
print('maxtile', 'Y:', maxtile.tms_x, 'Y:', maxtile.tms_y, 'zoom:', maxtile.zoom)
66+
67+
mintms_x, mintms_y = mintile.tms_x, mintile.tms_y
68+
maxtms_x, maxtms_y = maxtile.tms_x, maxtile.tms_y
69+
70+
create_image_path(rootpath, zoom)
71+
72+
imagelists = []
73+
for x in range(mintms_x, maxtms_x + 1):
74+
for y in range(maxtms_y, mintms_y + 1):
75+
savepath = './%s/%d/%d_%d.png'%(rootpath, zoom, x, y)
76+
tileurl = basetileurl + '&x=%d&y=%d&z=%d'%(x, y, zoom)
77+
imagelists.append((tileurl, savepath))
78+
79+
return imagelists
80+
81+
async def save_image_v2(url):
82+
tileurl, savepath = url[0], url[1]
83+
loop = asyncio.get_event_loop()
84+
try:
85+
response = await loop.run_in_executor(None, functools.partial(request.urlretrieve, url = tileurl, filename = savepath))
86+
print('---- PID:', os.getpid(), tileurl)
87+
except Exception as e:
88+
print(e)
89+
90+
def main():
91+
minlon, minlat = 109.227, - 20.196 # 矩形区域左下角坐标
92+
maxlon, maxlat = 117.182, - 25.5 # 矩形区域右上角坐标
93+
minzoom, maxzoom = 12, 12
94+
95+
basetileurl = 'http://webrd02.is.autonavi.com/appmaptile?lang=zh_cn&size=1&scale=1&style=8'
96+
# basetileurl = 'http://mt2.google.cn/vt/lyrs=m&hl=zh-CN&gl=cn&x=%d&y=%d&z=%d'%(x, y, zoom)
97+
# basetileurl = 'http://wprd04.is.autonavi.com/appmaptile?lang=zh_cn&size=1&style=7&x=%d&y=%d&z=%d&scl=1&ltype=11'%(x, y, zoom)
98+
rootpath = './tilefile'
99+
100+
imagelists = create_image_url_v2(minlon, minlat, maxlon, maxlat, minzoom, maxzoom, basetileurl, rootpath)
101+
102+
tasks = [asyncio.ensure_future(save_image_v2(url)) for url in imagelists]
103+
loop = asyncio.get_event_loop()
104+
loop.run_until_complete(asyncio.wait(tasks))
105+
loop.close()
106+
107+
if __name__ == '__main__':
108+
main_start = time.time()
109+
main()
110+
print('---- All time: ', time.time() - main_start)

‎offline_tileserver/multiprocess_tilemap.py

+71
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
## 多进程爬取切片地图
2+
from pygeotile.tile import Tile
3+
from urllib import request
4+
import os
5+
from PIL import Image
6+
from multiprocessing import Pool, TimeoutError, Queue
7+
8+
def create_image_path(rootpath, level):
9+
path = './%s/%d'%(rootpath, level)
10+
if not os.path.exists(path):
11+
os.makedirs(path)
12+
return path
13+
14+
def create_image_url(minlon, minlat, maxlon, maxlat, minzoom, maxzoom, basetileurl, rootpath):
15+
for zoom in range(minzoom, maxzoom + 1):
16+
mintile = Tile.for_latitude_longitude(latitude = minlat, longitude = minlon, zoom = zoom)
17+
maxtile = Tile.for_latitude_longitude(latitude = maxlat, longitude = maxlon, zoom = zoom)
18+
19+
print('mintile', 'X:', mintile.tms_x, 'Y:', mintile.tms_y, 'zoom:', mintile.zoom)
20+
print('maxtile', 'Y:', maxtile.tms_x, 'Y:', maxtile.tms_y, 'zoom:', maxtile.zoom)
21+
22+
mintms_x, mintms_y = mintile.tms_x, mintile.tms_y
23+
maxtms_x, maxtms_y = maxtile.tms_x, maxtile.tms_y
24+
25+
create_image_path(rootpath, zoom)
26+
27+
global imagelists
28+
imagelists = Queue()
29+
for x in range(mintms_x, maxtms_x + 1):
30+
for y in range(maxtms_y, mintms_y + 1):
31+
savepath = './%s/%d/%d_%d.png'%(rootpath, zoom, x, y)
32+
tileurl = basetileurl + '&x=%d&y=%d&z=%d'%(x, y, zoom)
33+
imagelists.put((tileurl, savepath))
34+
35+
return imagelists
36+
37+
def save_image(image):
38+
try:
39+
tileurl, savepath = image[0], image[1]
40+
request.urlretrieve(tileurl, savepath)
41+
print('---- PID:', os.getpid(), tileurl)
42+
except Exception as e:
43+
print('---- {}. Error: {}'.format(tileurl, e))
44+
with open('./multiprocess_error.log', 'a') as f:
45+
f.write('---- {}. Error: {}'.format(tileurl, e))
46+
47+
if __name__ == '__main__':
48+
minlon, minlat = 109.227, - 20.196 # 矩形区域左下角坐标
49+
maxlon, maxlat = 117.182, - 25.5 # 矩形区域右上角坐标
50+
minzoom, maxzoom = 12, 12
51+
52+
basetileurl = 'http://webrd02.is.autonavi.com/appmaptile?lang=zh_cn&size=1&scale=1&style=8'
53+
# basetileurl = 'http://mt2.google.cn/vt/lyrs=m&hl=zh-CN&gl=cn&x=%d&y=%d&z=%d'%(x, y, zoom)
54+
# basetileurl = 'http://wprd04.is.autonavi.com/appmaptile?lang=zh_cn&size=1&style=7&x=%d&y=%d&z=%d&scl=1&ltype=11'
55+
rootpath = './tilefile'
56+
57+
imagelists = create_image_url(minlon, minlat, maxlon, maxlat, minzoom, maxzoom, basetileurl, rootpath)
58+
print('---- Init size', imagelists.qsize())
59+
p = Pool(processes = 4)
60+
while True:
61+
print('---- Length', imagelists.qsize())
62+
if imagelists.empty(): #为空退出循环结束线程
63+
print("Done!")
64+
break
65+
image = imagelists.get()
66+
p.apply_async(save_image, args = (image, ))
67+
68+
p.close()
69+
p.join()
70+
used_time = time.time() - start_time
71+
print('---- Used time: ', used_time)

‎offline_tileserver/multithread_tilemap.py

+81
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
## 多线程爬取切片地图
2+
from pygeotile.tile import Tile
3+
from urllib import request
4+
import os
5+
import time
6+
from multiprocessing import Pool, TimeoutError, Queue
7+
import threading
8+
9+
global start_time
10+
start_time = time.time()
11+
12+
def create_image_path(rootpath, level):
13+
path = './%s/%d'%(rootpath, level)
14+
if not os.path.exists(path):
15+
os.makedirs(path)
16+
return path
17+
18+
def create_image_url(minlon, minlat, maxlon, maxlat, minzoom, maxzoom, basetileurl, rootpath):
19+
for zoom in range(minzoom, maxzoom + 1):
20+
mintile = Tile.for_latitude_longitude(latitude = minlat, longitude = minlon, zoom = zoom)
21+
maxtile = Tile.for_latitude_longitude(latitude = maxlat, longitude = maxlon, zoom = zoom)
22+
23+
print('mintile', 'X:', mintile.tms_x, 'Y:', mintile.tms_y, 'zoom:', mintile.zoom)
24+
print('maxtile', 'Y:', maxtile.tms_x, 'Y:', maxtile.tms_y, 'zoom:', maxtile.zoom)
25+
26+
mintms_x, mintms_y = mintile.tms_x, mintile.tms_y
27+
maxtms_x, maxtms_y = maxtile.tms_x, maxtile.tms_y
28+
29+
create_image_path(rootpath, zoom)
30+
31+
global imagelists
32+
imagelists = Queue()
33+
for x in range(mintms_x, maxtms_x + 1):
34+
for y in range(maxtms_y, mintms_y + 1):
35+
savepath = './%s/%d/%d_%d.png'%(rootpath, zoom, x, y)
36+
tileurl = basetileurl + '&x=%d&y=%d&z=%d'%(x, y, zoom)
37+
imagelists.put((tileurl, savepath))
38+
39+
return imagelists
40+
41+
def save_image(imagelists):
42+
while True:
43+
try:
44+
print('---- Length', imagelists.qsize())
45+
if imagelists.empty():
46+
print("Done!")
47+
global used_time
48+
used_time = time.time() - start_time
49+
print('---- Used time: ', used_time)
50+
break
51+
image = imagelists.get()
52+
tileurl, savepath = image[0], image[1]
53+
request.urlretrieve(tileurl, savepath)
54+
print('---- PID:', os.getpid(), tileurl)
55+
except Exception as e:
56+
print('---- Error: {}'.format(e))
57+
with open('./error.log', 'a') as f:
58+
f.write('---- Error: {}'.format(e))
59+
60+
def main():
61+
minlon, minlat = 109.227, - 20.196 # 矩形区域左下角坐标
62+
maxlon, maxlat = 117.182, - 25.5 # 矩形区域右上角坐标
63+
minzoom, maxzoom = 12, 12
64+
65+
basetileurl = 'http://webrd02.is.autonavi.com/appmaptile?lang=zh_cn&size=1&scale=1&style=8'
66+
# basetileurl = 'http://mt2.google.cn/vt/lyrs=m&hl=zh-CN&gl=cn&x=%d&y=%d&z=%d'%(x, y, zoom)
67+
# basetileurl = 'http://wprd04.is.autonavi.com/appmaptile?lang=zh_cn&size=1&style=7&x=%d&y=%d&z=%d&scl=1&ltype=11'%(x, y, zoom)
68+
rootpath = './tilefile'
69+
70+
global threadNum
71+
threadNum = 10
72+
73+
imagelists = create_image_url(minlon, minlat, maxlon, maxlat, minzoom, maxzoom, basetileurl, rootpath)
74+
print('---- Init size', imagelists.qsize())
75+
for i in range(threadNum):
76+
td = threading.Thread(target = save_image, args = (imagelists, ))
77+
td.start()
78+
td.stop()
79+
80+
if __name__ == '__main__':
81+
main()

0 commit comments

Comments
 (0)