添加代理，防止爬虫被办

Author: awolfly9

.gitignore

@@ -94,7 +94,7 @@ ENV/
 *.DS_Store
 
 # custom dir
-Plugins
-other
-plugin
-log
+Plugins/
+other/
+plugin/
+log/

UnityAssetStore/middlewares.py

@@ -8,27 +8,16 @@
 class ProxyMiddleware(object):
     def process_request(self, request, spider):
         try:
-            contents = requests.get('http://127.0.0.1:8000')
+            r = requests.get('http://101.200.55.192:8000/?type=1&count=20')
             #logging.info('contents:%s' % contents.text)
-            data = json.loads(contents.text)
-            proxy = random.choice(data)
-            ip = proxy.get('ip')
-            port = proxy.get('port')
-            address = str(ip) + ':' + str(port)
-            #request.meta['proxy'] = "http://%s" % proxy['ip_port']
-            request.meta['proxy'] = 'http://%s' % address
-            logging.info('********ProxyMiddleware proxy*******:%s' % request.meta['proxy'] )
+            data = json.loads(r.text)
+            if len(data) > 0:
+                proxy = random.choice(data)
+                ip = proxy.get('ip')
+                port = proxy.get('port')
+                address = '%s:%s' % (ip, port)
+                #request.meta['proxy'] = "http://%s" % proxy['ip_port']
+                request.meta['proxy'] = 'http://%s' % address
+                logging.info('********ProxyMiddleware proxy*******:%s' % request.meta['proxy'] )
         except:
-            pass
-
-
-#
-#
-# PROXIES = [
-#     {'ip_port': '111.11.228.75:80', 'user_pass': ''},
-#     {'ip_port': '120.198.243.22:80', 'user_pass': ''},
-#     {'ip_port': '111.8.60.9:8123', 'user_pass': ''},
-#     {'ip_port': '101.71.27.120:80', 'user_pass': ''},
-#     {'ip_port': '122.96.59.104:80', 'user_pass': ''},
-#     {'ip_port': '122.224.249.122:8088', 'user_pass': ''},
-# ]
+            pass

UnityAssetStore/settings.py

@@ -58,7 +58,6 @@
     'scrapy.contrib.downloadermiddleware.httpproxy.HttpProxyMiddleware': 1020,
     'UnityAssetStore.middlewares.ProxyMiddleware': 100,
 }
-
 # Enable or disable extensions
 # See http://scrapy.readthedocs.org/en/latest/topics/extensions.html
 #EXTENSIONS = {