From b2cad7976d1144b2bc273b538757ba3c97233761 Mon Sep 17 00:00:00 2001 From: Dan Buch Date: Sun, 5 Feb 2012 19:09:56 -0500 Subject: [PATCH 1/8] Adding timewarp script used to adjust upload dates in bulk. Very much geared toward solving the problem at hand. --- timewarp.py | 78 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 timewarp.py diff --git a/timewarp.py b/timewarp.py new file mode 100644 index 0000000..1db08ed --- /dev/null +++ b/timewarp.py @@ -0,0 +1,78 @@ +from __future__ import print_function + +import argparse +import json +import sys +from os.path import expanduser + +import flickrapi + + +class TimeWarper(object): + + def __init__(self, sysargs=sys.argv[:]): + self.sysargs = sysargs + parser = argparse.ArgumentParser() + parser.add_argument('min_upload_date') + parser.add_argument('max_upload_date') + parser.add_argument('set_to_date') + + parser.add_argument( + '-c', '--rcfile', default=expanduser('~/.frupplerc'), + type=argparse.FileType('r') + ) + + self.args = parser.parse_args(self.sysargs[1:]) + self._rc_conf = json.load(self.args.rcfile) + + self.flickr = flickrapi.FlickrAPI( + self._rc_conf['APIKEY'], self._rc_conf['APISECRET'] + ) + + token, frob = self.flickr.get_token_part_one(perms='write') + if not token: + raw_input("Press ENTER after you authorized this program") + + self.flickr.get_token_part_two((token, frob)) + + def set_upload_date_for_photos(self, pagenum, photo_nodes, to_date): + for i, photo in enumerate(photo_nodes): + setdate_response = self.flickr.photos_setDates( + photo_id=photo.attrib['id'], + date_posted=to_date + ) + print('page {} photo {}: rsp={}, attribs={}'.format( + pagenum, i, setdate_response, setdate_response.attrib)) + + def run(self): + query_args = dict( + user_id='me', + min_upload_date=self.args.min_upload_date, + max_upload_date=self.args.max_upload_date, + page=1 + ) + + print('Getting first page') + firstpage = self.flickr.photos_search(**query_args) + self.set_upload_date_for_photos( + 1, + firstpage.getchildren()[0].getchildren(), + self.args.set_to_date + ) + + npages = int(firstpage.getchildren()[0].attrib['pages']) + + for pagenum in range(1, npages + 1): + print('Getting another page') + page = self.flickr.photos_search(**query_args) + self.set_upload_date_for_photos( + pagenum, + page.getchildren()[0].getchildren(), + self.args.set_to_date + ) + + return 0 + + +if __name__ == '__main__': + sys.exit(TimeWarper().run()) From 64a22993bc4089d79e0685718fba2e0d285e3083 Mon Sep 17 00:00:00 2001 From: Dan Buch Date: Sun, 5 Feb 2012 19:12:32 -0500 Subject: [PATCH 2/8] Adding a silly little readme --- README.rst | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 README.rst diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..00ef6fb --- /dev/null +++ b/README.rst @@ -0,0 +1,5 @@ +============== +Flickr Scripts +============== + +Miscellaneous crap for doing stuff with my Flickr photos. From ab492e70d0407b98c9c639db171cfaa4a1a21390 Mon Sep 17 00:00:00 2001 From: Dan Buch Date: Sun, 5 Feb 2012 22:49:57 -0500 Subject: [PATCH 3/8] Now with less embarrassment! --- flickrscripts.py | 11 ++++ timewarp.py | 138 ++++++++++++++++++++++++++--------------------- 2 files changed, 88 insertions(+), 61 deletions(-) create mode 100644 flickrscripts.py diff --git a/flickrscripts.py b/flickrscripts.py new file mode 100644 index 0000000..1bbb7a9 --- /dev/null +++ b/flickrscripts.py @@ -0,0 +1,11 @@ +import flickrapi + + +def setup_flickr(self, api_key, api_secret): + self.flickr = flickrapi.FlickrAPI(api_key, api_secret) + + token, frob = self.flickr.get_token_part_one(perms='write') + if not token: + raw_input("Press ENTER after you authorized this program") + + self.flickr.get_token_part_two((token, frob)) diff --git a/timewarp.py b/timewarp.py index 1db08ed..8ebc7cf 100644 --- a/timewarp.py +++ b/timewarp.py @@ -1,78 +1,94 @@ -from __future__ import print_function - import argparse import json +import logging import sys +from datetime import datetime, timedelta from os.path import expanduser -import flickrapi +from flickrscripts import setup_flickr class TimeWarper(object): + flickr = None + _base_query_args = dict(user_id='me', page=1) - def __init__(self, sysargs=sys.argv[:]): - self.sysargs = sysargs - parser = argparse.ArgumentParser() - parser.add_argument('min_upload_date') - parser.add_argument('max_upload_date') - parser.add_argument('set_to_date') + def __init__(self, api_key, api_secret): + setup_flickr(self, api_key, api_secret) + self.log = logging.getLogger(self.__class__.__name__) + self.log.level = logging.INFO - parser.add_argument( - '-c', '--rcfile', default=expanduser('~/.frupplerc'), - type=argparse.FileType('r') + def run(self, set_to_date, **query_args): + self.log.info( + 'Setting upload dates for photos matching query %r', query_args ) + query = self._base_query_args.copy() + query.update(query_args) - self.args = parser.parse_args(self.sysargs[1:]) - self._rc_conf = json.load(self.args.rcfile) - - self.flickr = flickrapi.FlickrAPI( - self._rc_conf['APIKEY'], self._rc_conf['APISECRET'] - ) - - token, frob = self.flickr.get_token_part_one(perms='write') - if not token: - raw_input("Press ENTER after you authorized this program") - - self.flickr.get_token_part_two((token, frob)) - - def set_upload_date_for_photos(self, pagenum, photo_nodes, to_date): - for i, photo in enumerate(photo_nodes): - setdate_response = self.flickr.photos_setDates( - photo_id=photo.attrib['id'], - date_posted=to_date - ) - print('page {} photo {}: rsp={}, attribs={}'.format( - pagenum, i, setdate_response, setdate_response.attrib)) - - def run(self): - query_args = dict( - user_id='me', - min_upload_date=self.args.min_upload_date, - max_upload_date=self.args.max_upload_date, - page=1 - ) - - print('Getting first page') - firstpage = self.flickr.photos_search(**query_args) - self.set_upload_date_for_photos( - 1, - firstpage.getchildren()[0].getchildren(), - self.args.set_to_date - ) - - npages = int(firstpage.getchildren()[0].attrib['pages']) - - for pagenum in range(1, npages + 1): - print('Getting another page') - page = self.flickr.photos_search(**query_args) - self.set_upload_date_for_photos( - pagenum, - page.getchildren()[0].getchildren(), - self.args.set_to_date - ) + for i, page in self._get_pages(query): + self.log.info('Processing page %s, attributes=%r', + i, page.attrib) + self._set_upload_date_for_photos(i, page, set_to_date) return 0 + def _set_upload_date_for_photos(self, pagenum, page, set_to_date): + for i, photo in enumerate(page.getchildren()[0].getchildren()): + setdate_response = self.flickr.photos_setDates( + photo_id=photo.attrib['id'], + date_posted=set_to_date + ) + self.log.info('page %s photo %s: response attribs=%s', + pagenum, i, setdate_response.attrib) + + def _get_pages(self, query): + self.log.info('Getting pages for query %r', query) + i = 1 + while True: + page = self.flickr.photos_search(**query) + if len(page.getchildren()[0].getchildren()): + yield i, page + i += 1 + else: + raise StopIteration + + +def main(sysargs=sys.argv[:]): + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument('set_to_date') + parser.add_argument('-q', '--search-query-param', default=[], + action='append', help='Add a key=value param to the search query list') + + parser.add_argument( + '-c', '--rcfile', default=expanduser('~/.flickrscripts.json'), + help='JSON file containing `APIKEY` and `APISECRET` keys', + type=argparse.FileType('r') + ) + + args = parser.parse_args(sysargs[1:]) + rc_conf = json.load(args.rcfile) + + logging.basicConfig(level=logging.INFO, stream=sys.stdout) + log = logging.getLogger('main') + log.level = logging.INFO + log.info('Preparing query args') + + query_args = {} + for key_val in args.search_query_param: + key, value = key_val.split('=', 1) + query_args[key] = value + + if len(query_args.keys()) == 0: + query_args = dict( + min_upload_date=(datetime.now() + timedelta(days=-1)), + max_upload_date=(datetime.now() + timedelta(days=1)) + ) + log.warn('No query arguments provided! ' + + 'Defaulting to %r', query_args) + + warper = TimeWarper(rc_conf['APIKEY'], rc_conf['APISECRET']) + return warper.run(args.set_to_date, **query_args) + if __name__ == '__main__': - sys.exit(TimeWarper().run()) + sys.exit(main()) From 7addaa6552f594c77a57d5a2a80c0a077aaba8e5 Mon Sep 17 00:00:00 2001 From: Dan Buch Date: Mon, 6 Feb 2012 08:02:20 -0500 Subject: [PATCH 4/8] Making flickr client setup func more sane, adding convenience func for futzing in the repl --- flickrscripts.py | 17 +++++++++++++---- timewarp.py | 2 +- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/flickrscripts.py b/flickrscripts.py index 1bbb7a9..9761511 100644 --- a/flickrscripts.py +++ b/flickrscripts.py @@ -1,11 +1,20 @@ +import json +from os.path import expanduser + import flickrapi -def setup_flickr(self, api_key, api_secret): - self.flickr = flickrapi.FlickrAPI(api_key, api_secret) +def setup_flickr(api_key, api_secret): + flickr = flickrapi.FlickrAPI(api_key, api_secret) - token, frob = self.flickr.get_token_part_one(perms='write') + token, frob = flickr.get_token_part_one(perms='write') if not token: raw_input("Press ENTER after you authorized this program") - self.flickr.get_token_part_two((token, frob)) + flickr.get_token_part_two((token, frob)) + return flickr + + +def get_flickr_from_rc_file(rc_file=expanduser('~/.flickrscripts.json')): + rc_conf = json.load(open(rc_file)) + return setup_flickr(rc_conf['APIKEY'], rc_conf['APISECRET']) diff --git a/timewarp.py b/timewarp.py index 8ebc7cf..4edb832 100644 --- a/timewarp.py +++ b/timewarp.py @@ -13,7 +13,7 @@ class TimeWarper(object): _base_query_args = dict(user_id='me', page=1) def __init__(self, api_key, api_secret): - setup_flickr(self, api_key, api_secret) + self.flickr = setup_flickr(api_key, api_secret) self.log = logging.getLogger(self.__class__.__name__) self.log.level = logging.INFO From 7ec57759fa8fe86b9920c3c691f0f9c28f156cdc Mon Sep 17 00:00:00 2001 From: Dan Buch Date: Mon, 6 Feb 2012 08:07:51 -0500 Subject: [PATCH 5/8] Fixing sillyness with fetching all photos. Weirdness encountered earlier with `.findall` must have been my imagination. --- timewarp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/timewarp.py b/timewarp.py index 4edb832..901e782 100644 --- a/timewarp.py +++ b/timewarp.py @@ -32,7 +32,7 @@ class TimeWarper(object): return 0 def _set_upload_date_for_photos(self, pagenum, page, set_to_date): - for i, photo in enumerate(page.getchildren()[0].getchildren()): + for i, photo in enumerate(page.findall('photos/photo'): setdate_response = self.flickr.photos_setDates( photo_id=photo.attrib['id'], date_posted=set_to_date @@ -45,7 +45,7 @@ class TimeWarper(object): i = 1 while True: page = self.flickr.photos_search(**query) - if len(page.getchildren()[0].getchildren()): + if len(page.findall('photos/photo')): yield i, page i += 1 else: From f0381040dc938922e123c9696da9896bd6bca411 Mon Sep 17 00:00:00 2001 From: Dan Buch Date: Mon, 6 Feb 2012 08:38:06 -0500 Subject: [PATCH 6/8] Starting to fill in bits for detecting duplicate uploads --- flickrscripts.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/flickrscripts.py b/flickrscripts.py index 9761511..8785ffa 100644 --- a/flickrscripts.py +++ b/flickrscripts.py @@ -1,4 +1,5 @@ import json +from hashlib import sha1 from os.path import expanduser import flickrapi @@ -15,6 +16,30 @@ def setup_flickr(api_key, api_secret): return flickr +def get_photo_signature(flickr, photo_id): + info = flickr.photos_getInfo(photo_id=photo_id).find('photo') + exif = flickr.photos_getExif(photo_id=photo_id).find('photo') + + if not info or not exif: + raise InvalidPhotoIdError(photo_id) + + return [ + info.get('originalformat'), info.find('owner').get('location'), + info.find('dates').get('taken') + ] + sorted( + (e.get('label'), e.get('tag'), e.get('tagspace'), e.get('tagspaceid'), + e.find('raw').text) for e in exif.findall('exif') + ) + + +def get_photo_sha1sum(flickr, photo_id): + return sha1(str(get_photo_signature(flickr, photo_id))) + + def get_flickr_from_rc_file(rc_file=expanduser('~/.flickrscripts.json')): rc_conf = json.load(open(rc_file)) return setup_flickr(rc_conf['APIKEY'], rc_conf['APISECRET']) + + +class InvalidPhotoIdError(ValueError): + pass From c4122c364362542ced5f304dbfd6eb714cb1a935 Mon Sep 17 00:00:00 2001 From: Dan Buch Date: Mon, 6 Feb 2012 09:44:30 -0500 Subject: [PATCH 7/8] Moving more toward being an actual library/script thing... mostly for practice since I'm no longer doing Python full-time --- .gitignore | 1 + flickrscripts/__init__.py | 29 +++++++++++++++++++++ flickrscripts.py => flickrscripts/common.py | 0 timewarp.py => flickrscripts/timewarp.py | 7 ++--- setup.py | 10 +++++++ 5 files changed, 44 insertions(+), 3 deletions(-) create mode 100644 .gitignore create mode 100644 flickrscripts/__init__.py rename flickrscripts.py => flickrscripts/common.py (100%) rename timewarp.py => flickrscripts/timewarp.py (93%) create mode 100644 setup.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..11041c7 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +*.egg-info diff --git a/flickrscripts/__init__.py b/flickrscripts/__init__.py new file mode 100644 index 0000000..68d857f --- /dev/null +++ b/flickrscripts/__init__.py @@ -0,0 +1,29 @@ +from __future__ import print_function + +import sys +from os.path import basename + + +__meta__ = dict( + name='flickrscripts', + version='0.1.0', + entry_points={ + 'console_scripts': [ + 'flickrscripts = flickrscripts:route', + ], + }, +) +USAGE = 'Usage: {prog} [args]' + + +def route(sysargs=sys.argv[:]): + try: + subcommand = sysargs.pop(1) + import_name = 'flickrscripts.{}'.format(subcommand) + + module = __import__(import_name, fromlist=[import_name]) + return module.main(sysargs) + except IndexError: + print(USAGE.format(prog=basename(sysargs[0])), file=sys.stderr) + print('You must provide a subcommand', file=sys.stderr) + return 1 diff --git a/flickrscripts.py b/flickrscripts/common.py similarity index 100% rename from flickrscripts.py rename to flickrscripts/common.py diff --git a/timewarp.py b/flickrscripts/timewarp.py similarity index 93% rename from timewarp.py rename to flickrscripts/timewarp.py index 901e782..b76e094 100644 --- a/timewarp.py +++ b/flickrscripts/timewarp.py @@ -5,7 +5,7 @@ import sys from datetime import datetime, timedelta from os.path import expanduser -from flickrscripts import setup_flickr +from flickrscripts.common import setup_flickr class TimeWarper(object): @@ -32,7 +32,7 @@ class TimeWarper(object): return 0 def _set_upload_date_for_photos(self, pagenum, page, set_to_date): - for i, photo in enumerate(page.findall('photos/photo'): + for i, photo in enumerate(page.findall('photos/photo')): setdate_response = self.flickr.photos_setDates( photo_id=photo.attrib['id'], date_posted=set_to_date @@ -54,7 +54,8 @@ class TimeWarper(object): def main(sysargs=sys.argv[:]): parser = argparse.ArgumentParser( - formatter_class=argparse.ArgumentDefaultsHelpFormatter) + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + prog='flickrscripts timewarp') parser.add_argument('set_to_date') parser.add_argument('-q', '--search-query-param', default=[], action='append', help='Add a key=value param to the search query list') diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..84b1a72 --- /dev/null +++ b/setup.py @@ -0,0 +1,10 @@ +import sys + +from setuptools import setup + +from flickrscripts import __meta__ + + +if __name__ == '__main__': + setup(**__meta__) + sys.exit(0) From 89393c390fe69e66bd2d9aa275ba652e320a3215 Mon Sep 17 00:00:00 2001 From: Dan Buch Date: Mon, 6 Feb 2012 09:50:03 -0500 Subject: [PATCH 8/8] Retroactive TDD! :-P --- flickrscripts/__init__.py | 1 + flickrscripts/common.py | 4 ++++ tests/itest_timewarp.py | 19 +++++++++++++++++++ 3 files changed, 24 insertions(+) create mode 100644 tests/itest_timewarp.py diff --git a/flickrscripts/__init__.py b/flickrscripts/__init__.py index 68d857f..eb8947f 100644 --- a/flickrscripts/__init__.py +++ b/flickrscripts/__init__.py @@ -12,6 +12,7 @@ __meta__ = dict( 'flickrscripts = flickrscripts:route', ], }, + test_suite='nose.collector', ) USAGE = 'Usage: {prog} [args]' diff --git a/flickrscripts/common.py b/flickrscripts/common.py index 8785ffa..cb2d39c 100644 --- a/flickrscripts/common.py +++ b/flickrscripts/common.py @@ -41,5 +41,9 @@ def get_flickr_from_rc_file(rc_file=expanduser('~/.flickrscripts.json')): return setup_flickr(rc_conf['APIKEY'], rc_conf['APISECRET']) +def load_rc_file(rc_file=expanduser('~/.flickrscripts.json')): + return json.load(open(rc_file)) + + class InvalidPhotoIdError(ValueError): pass diff --git a/tests/itest_timewarp.py b/tests/itest_timewarp.py new file mode 100644 index 0000000..248ab2c --- /dev/null +++ b/tests/itest_timewarp.py @@ -0,0 +1,19 @@ +import unittest + +from flickrscripts.timewarp import TimeWarper +from flickrscripts.common import load_rc_file + + +class TimeWarperTestCase(unittest.TestCase): + + def setUp(self): + rc_conf = load_rc_file() + self.warper = TimeWarper(rc_conf['APIKEY'], rc_conf['APISECRET']) + + def test_is_not_horribly_busted(self): + self.assertTrue(True, 'please forgive the horrible test') + + + +if __name__ == '__main__': + unittest.main()