Fixed tests, resolved conflicts

Author: Julian Zimmermann

.gitignore

@@ -9,7 +9,7 @@
 config/prod.cfg
 config/client_secrets.json
 config/auth_token.json
-config/lastNewsletterUrl.csv
+config/lastNewsletterUrl
 
 ebooks/
 node_modules/

config/dev.cfg

@@ -6,6 +6,7 @@ url.loginPostNewsletter=/loginPostNewsletter.html
 url.account=/myEbooks.html
 # params: 0=id, 1=format
 url.download=/ebook_download/{0}/{1}
+url.bookFromNewsletter=http://localhost:8080/urlFromNewsletter.html
 
 #time in seconds
 [delay]

dev/public/loginPostNewsletter.html

@@ -16,7 +16,29 @@
     "currency_symbol": "\u20ac"
   };
 </script>
-
+<form accept-charset="UTF-8" method="post" id="packt-user-login-form">
+  <div>
+    <div id="login-form" class="cf">
+      <div id="login-form-email">
+        <div class="form-item" id="email-wrapper">
+          <input type="text" maxlength="128" name="email" id="email" size="24" value="" placeholder="E-mail address" class="form-text required" />
+        </div>
+      </div>
+      <div id="login-form-pass">
+        <div class="form-item" id="password-wrapper">
+          <input type="password" name="password" id="password" maxlength="128" size="24" placeholder="Password" class="form-text required" />
+        </div>
+      </div>
+      <div id="login-form-submit">
+        <input type="submit" name="op" id="edit-submit-1" value="Login" class=" form-submit" />
+      </div>
+      <br />
+    </div>
+    <a href="https://www.packtpub.com/account/password">Forgotten your password?</a>
+    <input type="hidden" name="form_build_id" id="form-aaaaaaaaaa1111111111bbbbbbbbbb22" value="form-aaaaaaaaaa1111111111bbbbbbbbbb22" />
+    <input type="hidden" name="form_id" id="edit-packt-user-login-form" value="packt_user_login_form" />
+  </div>
+</form>
 <div id="main-book" class="cf promo" itemscope itemtype="http://schema.org/Book">
     <div class="book-top-block-wrapper cf">
         <div class="cf section-inner">

dev/public/urlFromNewsletter.html

@@ -0,0 +1 @@
+http://localhost:8080/loginPostNewsletter.html

script/packtpub.py

@@ -37,13 +37,11 @@ def __log_response(self, response, method='GET', detail=False):
             log_dict(response.headers)
 
     def __GET_login(self, url):
-        if self.__dev:
-            url = self.__url_base + self.__config.get('url', 'url.loginGet')
-
         response = self.__session.get(url, headers=self.__headers)
         self.__log_response(response, 'GET', self.__dev)
 
         soup = make_soup(response)
+
         form = soup.find('form', {'id': 'packt-user-login-form'})
         self.info['form_build_id'] = form.find('input', attrs={'name': 'form_build_id'})['value']
         self.info['form_id'] = form.find('input', attrs={'name': 'form_id'})['value']
@@ -57,7 +55,6 @@ def __POST_login(self, url):
 
         response = None
         if self.__dev:
-            url = self.__url_base + self.__config.get('url', 'url.loginPost')
             response = self.__session.get(url, headers=self.__headers, data=data)
             self.__log_response(response, 'GET', self.__dev)
         else:
@@ -124,10 +121,17 @@ def __GET_claim(self):
     def runDaily(self):
         """
         """
+        if self.__dev:
+            loginUrl = self.__url_base + self.__config.get('url', 'url.loginGet')
+        else:
+            loginUrl = self.__url_base + self.__config.get('url', 'url.login')
 
-        loginUrl = self.__url_base + self.__config.get('url', 'url.login')
         self.__GET_login(loginUrl)
         wait(self.__delay, self.__dev)
+
+        if self.__dev:
+            loginUrl = self.__url_base + self.__config.get('url', 'url.loginPost')
+
         soup = self.__POST_login(loginUrl)
         self.__parseDailyBookInfo(soup)
         wait(self.__delay, self.__dev)

script/spider.py

@@ -101,10 +101,10 @@ def main():
 
         if os.path.isfile(lastNewsletterUrlPath):
             with open(lastNewsletterUrlPath, 'r+') as f:
-                lastNewsletterUrl = f.read()
+                lastNewsletterUrl = f.read().strip()
 
         # the default URL is generated by an Google apps script, see README for details and self-hosting
-        currentNewsletterUrl = requests.get(config.get('url', 'url.bookFromNewsletter')).text
+        currentNewsletterUrl = requests.get(config.get('url', 'url.bookFromNewsletter')).text.strip()
 
         if lastNewsletterUrl != currentNewsletterUrl:
             log_info('[*] getting free ebook from newsletter')