Skip to content

proper subreddits abstraction and subredit whole timespan scan #32

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
fiocfun wants to merge 1 commit into
base: master
Choose a base branch
from
Open

proper subreddits abstraction and subredit whole timespan scan #32

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 17 additions & 1 deletion README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,24 @@ pip install pillow
Executing
=========

Execute `Gonewild.py` in the `./py/` directory. Include no arguments to start infinite loop which checks for and downloads new content. Other options available, see:
Execute `Gonewild.py` in the `./py/` directory.

Initial setup:
```bash
./py/Gonewild.py --reddit username password
./py/Gonewild.py --add-sub subreddit1,sub2
```
Include no arguments to start infinite loop which checks for and downloads new content.

Other options available, see:
```bash
python Gonewild.py --help
```

Upgrade from previous version

@4pr0n 4pr0n Oct 22, 2014

Copy link
Copy Markdown
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The DB.py file automatically creates tables if they don't already exist.

gonewilder/py/DB.py

Lines 133 to 137 in 21cf14e

# Don't create tables if not supplied.
if SCHEMA != None and SCHEMA != {} and len(SCHEMA) > 0:
# Create table for every schema given.
for key in SCHEMA:
self.create_table(key, SCHEMA[key])

=============================

```bash
sqlite3 ./database.db 'create table subs(id integer primary key autoincrement , sub text unique, sinceid text);' #updrade db
./py/Gonewild.py --add-sub oldsub,oldsub2 #readd old subs manually
```
82 changes: 61 additions & 21 deletions py/DB.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,14 +19,20 @@
'id integer primary key autoincrement, \n\t' +
'username text unique, \n\t' +
'sinceid text, \n\t' +
'created integer, \n\t' +
'created integer, \n\t' +
'updated integer, \n\t' +
'deleted integer, \n\t' +
'blacklist integer, \n\t' +
'views integer, \n\t' +
'rating integer, \n\t' +
'ratings integer \n\t',

'subs' :

@4pr0n 4pr0n Oct 22, 2014

Copy link
Copy Markdown
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Looks like you're using spaces instead of tabs.
I usually prefer spaces over tabs myself, but I can't merge this if it has mixed spaces and tabs. I'd prefer for the whole project to stay on the same code style.

'\n\t' +
'id integer primary key autoincrement, \n\t' +
'sub text unique, \n\t' +
'sinceid text \n\t',

'posts' :
'\n\t' +
'id text primary key, \n\t' +
Expand Down Expand Up @@ -57,7 +63,7 @@
'downs integer, \n\t' +
'foreign key(userid) references users(id)\n\t',

'albums' :
'albums' :
'\n\t'
'id integer primary key, \n\t' +
'path text unique, \n\t' +
Expand All @@ -76,7 +82,7 @@
'source text, \n\t' +
'width integer, \n\t' +
'height integer, \n\t' +
'size integer, \n\t' +
'size integer, \n\t' +
'thumb text, \n\t' +
'type text, \n\t' + # image/video
'albumid integer, \n\t' +
Expand All @@ -85,7 +91,7 @@
'views integer, \n\t' +
'foreign key(userid) references users(id), \n\t' +
'foreign key(albumid) references albums(id)\n\t',

'zips' :
'\n\t' +
'zippath text unique, \n\t' +
Expand Down Expand Up @@ -129,21 +135,21 @@ def __init__(self):
# Create table for every schema given.
for key in SCHEMA:
self.create_table(key, SCHEMA[key])

def debug(self, text):
tstamp = time.strftime('[%Y-%m-%dT%H:%M:%SZ]', time.gmtime())
text = '%s DB: %s' % (tstamp, text)
self.logger.write('%s\n' % text)
if self.logger != stderr:
stderr.write('%s\n' % text)

def create_table(self, table_name, schema):
cur = self.conn.cursor()
query = '''create table if not exists %s (%s)''' % (table_name, schema)
cur.execute(query)
self.commit()
cur.close()

def commit(self):
try_again = True
while try_again:
Expand All @@ -152,7 +158,7 @@ def commit(self):
try_again = False
except:
time.sleep(1)

def insert(self, table, values):
cur = self.conn.cursor()
try:
Expand All @@ -168,21 +174,21 @@ def insert(self, table, values):
except sqlite3.IntegrityError:
cur.close()
return -1

def delete(self, table, where, values=[]):
cur = self.conn.cursor()
q = '''
delete from %s
where %s
''' % (table, where)
cur.execute(q, values)

def get_cursor(self):
return self.conn.cursor()

def count(self, table, where='', values=[]):
return self.select_one('count(*)', table, where, values=values)

def select(self, what, table, where='', values=[]):
cur = self.conn.cursor()
query = '''
Expand Down Expand Up @@ -211,7 +217,7 @@ def select_one(self, what, table, where='', values=[]):
one = execur.fetchone()
cur.close()
return one[0]

def update(self, table, what, where='', values=[]):
cur = self.conn.cursor()
if where != '':
Expand Down Expand Up @@ -257,7 +263,7 @@ def add_user(self, user, new=False):
self.debug('add_user: user "%s" already exists in %susers: %s' % (user, 'new' if new else '', str(e)))
raise e
self.commit()

def remove_user(self, user):
userid = self.get_user_id(user)
user = self.select_one('username', 'users', where='id = ?', values=[userid])
Expand Down Expand Up @@ -325,7 +331,41 @@ def set_last_since_id(self, user, since_id):
''' % (since_id, user)
cur.execute(query)
self.commit()


def add_sub(self, sub):
cur = self.conn.cursor()
cur.execute('insert or ignore into subs(sub) values (?)', [sub])
self.commit()

def remove_sub(self, sub):
self.delete('subs', 'sub like ?', [sub])
self.commit()

def get_subs_list(self):
result = []
for sub in self.select('sub', 'subs'):
result.append(sub[0])
return result

def get_sub_last_since_id(self, sub):
cur = self.conn.cursor()
results = cur.execute('''
select sinceid
from subs
where sub like "%s"
''' % sub)
return results.fetchall()[0][0]

def set_sub_last_since_id(self, sub, since_id):
cur = self.conn.cursor()
query = '''
update subs
set sinceid = "%s"
where sub like "%s"
''' % (since_id, sub)
cur.execute(query)
self.commit()

def add_post(self, post, legacy=0):
userid = self.get_user_id(post.author)
values = [ (
Expand Down Expand Up @@ -515,8 +555,8 @@ def add_existing_image(self, user, oldimage, oldpath, subdir='', album_id=-1):
self.debug('add_existing_image: create_thumbnail failed: %s' % str(e))
thumbnail = path.join(ImageUtils.get_root(), 'images', 'nothumb.png')
try:
self.add_image(newimage, user, url,
dims[0], dims[1], size, thumbnail, 'image',
self.add_image(newimage, user, url,
dims[0], dims[1], size, thumbnail, 'image',
album_id, post, comment)
except Exception, e:
self.debug('add_existing_image: failed: %s' % str(e))
Expand Down Expand Up @@ -598,7 +638,7 @@ def add_existing_album(self, user, oldalbum, oldpath):
except Exception, e:
#self.debug('add_existing_image: %s' % str(e))
pass

def get_credentials(self, site):
if self.count('credentials', 'site = ?', [site]) == 0:
raise Exception('Credentials for %s not found in database, run "Gonewild.py --help" for more info' % site)
Expand Down Expand Up @@ -628,7 +668,7 @@ def set_credentials(self, site, username, password):
from traceback import format_exc
self.debug('\n%s' % format_exc())
raise e

def update_user(self, user):
cur = self.conn.cursor()
query = '''
Expand Down Expand Up @@ -671,7 +711,7 @@ def mark_as_deleted(self, user):

def already_friend(self, user):
return self.count('friends', 'username = ?', [user]) > 0

def add_friend(self, user):
cur = self.conn.cursor()
cur.execute('insert into friends values (?)', [user])
Expand Down Expand Up @@ -723,7 +763,7 @@ def set_config(self, key, value):
cur.close()
except Exception, e:
self.debug('failed to set config key "%s" to value "%s": %s' % (key, value, str(e)))


if __name__ == '__main__':
db = DB()
Expand Down
71 changes: 49 additions & 22 deletions py/Gonewild.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ def __init__(self):
self.db = DB() # Database instance
self.reddit = Reddit()
self.excluded_subs = self.db.get_excluded_subreddits()

def debug(self, text):
tstamp = strftime('[%Y-%m-%dT%H:%M:%SZ]', gmtime())
text = '%s Gonewild: %s' % (tstamp, text)
Expand Down Expand Up @@ -127,7 +127,7 @@ def poll_user(self, user):
continue

self.get_and_process_urls_from_child(child)

self.debug('%s: poll_user: done' % user)

# Set last 'since' to the most-recent post/comment ID
Expand Down Expand Up @@ -297,7 +297,7 @@ def process_url(self, url, url_index, child):
commid
)
self.db.update_user(child.author)

def infinite_loop(self):
users = self.db.get_users(new=False)

Expand Down Expand Up @@ -354,28 +354,47 @@ def infinite_loop(self):
except Exception, e:
self.debug('infinite_loop: poll_user: %s' % str(e))
from traceback import format_exc
print format_exc()
print format_exc()

def add_top_users(self):
users = []
subs = ['gonewild']
self.debug('add_top_users: loading top posts for the week from %s' % ','.join(subs))
try:
posts = self.reddit.get('http://www.reddit.com/r/%s/top.json?t=week' % '+'.join(subs))
except Exception, e:
self.debug('add_top_users: Exception: %s' % str(e))
return users
for post in posts:
if post.author == '[deleted]': continue
if not self.db.user_already_added(post.author):
self.debug('add_top_users: Found new user, adding /u/%s' % post.author)
self.db.add_user(post.author, new=True)
subusers = set()
for sub in self.db.get_subs_list():
subusers.update(self.poll_sub(sub))
users = []
for user in subusers:
if user == '[deleted]': continue
if not self.db.user_already_added(user):
self.debug('add_top_users: Found new user, adding /u/%s' % user)
self.db.add_user(user, new=True)
friend_zone = self.db.get_config('friend_zone')
if friend_zone == None or friend_zone == 'none':
self.add_friend(post.author)
users.append(post.author)
self.add_friend(user)
users.append(user)
return users

def poll_sub(self, sub):
since_id = self.db.get_sub_last_since_id(sub)
# Get posts/comments for user
self.debug('%s: poll_sub: since "%s"' % (sub, since_id))
try:
posts = self.reddit.get_sub(sub, since=since_id)
except Exception, e:
if '404: Not Found' in str(e):
# User is deleted, mark it as such
self.debug('%s: poll_sub: user is 404' % sub)
return
self.debug('%s: poll_sub: error %s' % (sub, str(e)))
return

if len(posts) == 0:
return set()

users = set([post.author for post in posts])
self.debug('%s: poll_sub: %d new posts found, %d posters' % (sub, len(posts), len(users)))
self.debug('%s: poll_sub: setting most-recent since_id to "%s"' % (sub, posts[0].id))
self.db.set_sub_last_since_id(sub, posts[0].id)
return users


def add_friend(self, user):
try:
Expand Down Expand Up @@ -441,7 +460,7 @@ def compare_friends(self, add_friends=False):
self.debug('Added /u/%s as a friend on reddit' % friend)
else:
self.debug('Found %d users that are not friended. to friend them, execute:\npython Gonewild.py --friend %s' % (len(need2add), ','.join(need2add)))

def toggle_addtop(self):
if self.db.get_config('add_top_users') != 'false':
self.db.set_config('add_top_users', 'false')
Expand Down Expand Up @@ -503,7 +522,7 @@ def setup_config(self):
'save_thumbnails' : 'true',
'add_top_users' : 'true',
'excluded_subreddits' : '',
'friend_zone' : 'some',
'friend_zone' : 'none',
'last_user' : ''
}
for (key,value) in keys.iteritems():
Expand All @@ -522,6 +541,9 @@ def handle_arguments(gw):
parser.add_argument('--add', '-a',
help='Add user(s) to scan for new content',
metavar='USER')
parser.add_argument('--add-sub',
help='Add sub(s) to scan for new content',
metavar='SUBREDDIT')
parser.add_argument('--add-top', '-tz',
help='Toggle adding top users from /r/gonewild',
action='store_true')
Expand Down Expand Up @@ -616,6 +638,12 @@ def handle_arguments(gw):
else:
gw.debug('Warning: User already added: /u/%s' % user)

elif args.add_sub:
subs = args.add_sub.replace('r/', '').replace('/', '').split(',')
for sub in subs:
gw.db.add_sub(sub)
gw.debug('Added sub: /r/%s' % sub)

elif args.friend:
users = args.friend.replace('u/', '').replace('/', '').split(',')
gw.login()
Expand Down Expand Up @@ -716,4 +744,3 @@ def handle_arguments(gw):

gw.login()
gw.infinite_loop()

Loading