From 298445b86467bcbd6f852c4d2600afb05ec58b68 Mon Sep 17 00:00:00 2001 From: "@gdorn@social.coop" Date: Fri, 1 Dec 2017 19:29:50 -0800 Subject: [PATCH 1/7] Added requirements.txt, including missing PyYAML requirement --- requirements.txt | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..2303a60 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,7 @@ +Flask==0.12.2 +bs4==0.0.1 +feedgenerator==1.9 +tweepy==3.5.0 +pytz==2017.3 +Mastodon.py==1.1.2 +PyYAML==3.12 From 2f46c5f98c079541528926b50d6f9dcb640b84c1 Mon Sep 17 00:00:00 2001 From: "@gdorn@social.coop" Date: Fri, 1 Dec 2017 19:31:06 -0800 Subject: [PATCH 2/7] Refactor for code reuse, update for mastodon client update (datetimes, timezones), add helpful script for creating minimum-permission mastodon client --- app.py | 89 +++++++++++++++++++++++++++++++++------ config.example.yml | 1 + config.py | 12 ++++++ create_mastodon_client.py | 36 ++++++++++++++++ 4 files changed, 125 insertions(+), 13 deletions(-) mode change 100644 => 100755 app.py mode change 100644 => 100755 config.example.yml create mode 100755 config.py create mode 100755 create_mastodon_client.py diff --git a/app.py b/app.py old mode 100644 new mode 100755 index 84abf80..6ebff75 --- a/app.py +++ b/app.py @@ -11,18 +11,13 @@ import tweepy import yaml import sys - +import os +from config import get_config app = Flask(__name__) app.debug = True - -with open('config.yml', 'r') as stream: - try: - param = yaml.safe_load(stream) - except yaml.YAMLError as e: - print(e) - sys.exit() +param = get_config() # Twitter try: @@ -40,9 +35,20 @@ # Mastodon try: + client_file = param['mastodon']['client_id_file'] + if not os.path.exists(client_file): + raise Exception("File not found: " + client_file) + access_token_file = param['mastodon']['access_token_file'] + if not os.path.exists(access_token_file): + raise Exception("File not found: " + client_file) + + mastodon_url = param['mastodon'].get('url', 'https://mastodon.social') + print("Using Mastodon instance at:", mastodon_url) + mastodon = Mastodon( - client_id=param['mastodon']['client_id_file'], - access_token=param['mastodon']['access_token_file'] + client_id=client_file, + access_token=access_token_file, + api_base_url=mastodon_url ) except Exception as e: print('Error Mastodon instance creation: ' + str(e)) @@ -189,7 +195,8 @@ def tootfeed(query_feed): '♻ : ' + str(toot['reblogs_count']) + ', ' + \ '✰ : ' + str(toot['favourites_count']) + '' - toot['created_at'] = datetime.datetime.strptime(toot['created_at'], '%Y-%m-%dT%H:%M:%S.%fZ') + if isinstance(toot['created_at'], str): + toot['created_at'] = datetime.datetime.strptime(toot['created_at'], '%Y-%m-%dT%H:%M:%S.%fZ') buffered.append(toot.copy()) @@ -204,12 +211,67 @@ def tootfeed(query_feed): for toot in buffered: text = BeautifulSoup(toot['content'], "html.parser").text + pubdate = toot['created_at'] + if not pubdate.tzinfo: + pubdate = utc.localize(pubdate).astimezone(pytz.timezone(param['feed']['timezone'])) + if len(text) > 100: text = text[:100] + '... ' f.add_item(title=toot['account']['display_name'] + ' (' + toot['account']['username'] + '): ' + text, link=toot['url'], - pubdate=utc.localize(toot['created_at']).astimezone(pytz.timezone(param['feed']['timezone'])), + pubdate=pubdate, + description=toot['htmltext']) + + xml = f.writeString('UTF-8') + else: + xml = 'error - Mastodon parameters not defined' + + return xml + +@app.route('/toot_favorites') +def toot_favorites_feed(): + """ generate an rss feed authenticated user's favorites """ + + if mastodonOK: + buffered = [] +# hashtagResult = mastodon.timeline_hashtag(query_feed) + favorite_toots = mastodon.favourites() + for toot in favorite_toots: + + toot['htmltext'] = '
' + toot['account']['display_name'] + \ + ' ' + toot['account']['username'] + \ + ': ' + toot['content'] + '
' + \ + '♻ : ' + str(toot['reblogs_count']) + ', ' + \ + '✰ : ' + str(toot['favourites_count']) + '
' + + if isinstance(toot['created_at'], str): + toot['created_at'] = datetime.datetime.strptime(toot['created_at'], '%Y-%m-%dT%H:%M:%S.%fZ') + + buffered.append(toot.copy()) + + utc = pytz.utc + f = feedgenerator.Rss201rev2Feed(title=param['mastodon']['title'] + ' Favourites ', + link=param['mastodon']['url'] + '/web/favourites', + description=param['mastodon']['description'], + language=param['feed']['language'], + author_name=param['feed']['author_name'], + feed_url=param['feed']['feed_url']) + + for toot in buffered: + + text = BeautifulSoup(toot['content'], "html.parser").text + pubdate = toot['created_at'] + if not pubdate.tzinfo: + pubdate = utc.localize(pubdate).astimezone(pytz.timezone(param['feed']['timezone'])) + + if len(text) > 100: + text = text[:100] + '... ' + f.add_item(title=toot['account']['display_name'] + ' (' + toot['account']['username'] + '): ' + + text, + link=toot['url'], + pubdate=pubdate, description=toot['htmltext']) xml = f.writeString('UTF-8') @@ -220,4 +282,5 @@ def tootfeed(query_feed): if __name__ == "__main__": - app.run() + app.run(use_reloader=True) + diff --git a/config.example.yml b/config.example.yml old mode 100644 new mode 100755 index 36f3aae..248a4f2 --- a/config.example.yml +++ b/config.example.yml @@ -8,6 +8,7 @@ mastodon: url: 'https://mastodon.social' client_id_file: 'tootrss_clientcred.txt' access_token_file: 'tootrss_usercred.txt' + app_name: 'tootrss' # Used to identify authenticated apps title: 'Recherche Mastodon : ' description: "Résultat d'une recherche Mastodon retournée dans un flux RSS." feed: diff --git a/config.py b/config.py new file mode 100755 index 0000000..bd67e20 --- /dev/null +++ b/config.py @@ -0,0 +1,12 @@ +""" +Loads and parses the configuration file. +""" +import yaml + +def get_config(): + with open('config.yml', 'r') as stream: + try: + return yaml.safe_load(stream) + except yaml.YAMLError as e: + print(e) + sys.exit() diff --git a/create_mastodon_client.py b/create_mastodon_client.py new file mode 100755 index 0000000..cc48d03 --- /dev/null +++ b/create_mastodon_client.py @@ -0,0 +1,36 @@ +from config import get_config +from mastodon import Mastodon +from getpass import getpass + +if __name__ == '__main__': + print("This script helps you create a new mastodon client and log in.") + print("Before we start, make sure config.yml exists.") + config = get_config() + + mast_cfg = config['mastodon'] + print("Configuration found.") + print("Looks like you want to use this instance:", mast_cfg['url']) + print("If that's wrong, now is a good time to cancel (^C) and fix it.") + input(" to continue") + + print("Registering a new app with {url} called {app_name} and saving credentials in {client_id_file}".format(**mast_cfg)) + + Mastodon.create_app(mast_cfg['app_name'], api_base_url=mast_cfg['url'], to_file=mast_cfg['client_id_file'], scopes=['read']) + mastodon = Mastodon(client_id=mast_cfg['client_id_file'], api_base_url=mast_cfg['url']) + print("Registration successful. Now to log in.") + user_email = input("User email: ") + password = getpass("Password (not shown and not saved):") + + # Log in - either every time, or use persisted + mastodon.log_in(user_email, password, to_file=mast_cfg['access_token_file'], scopes=['read']) + + print("Verifying credentials...") + try: + res = mastodon.account_verify_credentials() + print("Credentials look good; client reports user's account name is: " + res['acct']) + print("Configuration complete; app should appear at: " + mast_cfg['url'] + "/oauth/authorized_applications") + print("You should not need to log in again unless this app is removed or credentials expire.") + except Exception as ex: + print("Something went wrong; mastodon client reported an error:") + print(ex) + From 7fc1ee14176a509ba87db4aff6027da780f8af25 Mon Sep 17 00:00:00 2001 From: George Dorn Date: Sat, 2 Dec 2017 03:45:56 +0000 Subject: [PATCH 3/7] Docs for create_mastodon_client.py and url for favorite toots. --- README.md | 33 +++++++++------------------------ 1 file changed, 9 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 4526859..980a540 100644 --- a/README.md +++ b/README.md @@ -27,9 +27,9 @@ The RSS feed displays only the original tweets (not the retweets) and : ## **Steps :** -- install Python packages : flask, BeautifulSoup, Mastodon.py, feedgenerator, tweepy and pytz +- install Python packages : flask, BeautifulSoup, Mastodon.py, feedgenerator, tweepy, PyYAML and pytz ```bash -$ pip3 install flask bs4 feedgenerator tweepy pytz Mastodon.py +$ pip3 install -r requirements.txt ``` - clone this repo : @@ -37,31 +37,16 @@ $ pip3 install flask bs4 feedgenerator tweepy pytz Mastodon.py $ git clone https://github.com/SamR1/python-twootfeed.git ``` +- Copy the included **config.example.yml** to **config.yml** and fill in fields for the client(s) you will use. + - API Keys - for **Twitter** : see https://dev.twitter.com copy/paste the Twitter API key values in **config.yml.example** ('_consumerKey_' and '_consumerSecret_') - - for **Mastodon** : see [Python wrapper for the Mastodon API](https://github.com/halcy/Mastodon.py) - - generate client and user credentials files : - ```python - from mastodon import Mastodon - - # Register app - only once! - Mastodon.create_app( - 'pytooterapp', - to_file = 'tootrss_clientcred.txt' - ) - - # Log in - either every time, or use persisted - mastodon = Mastodon(client_id = 'tootrss_clientcred.txt') - mastodon.log_in( - 'my_login_email@example.com', - 'incrediblygoodpassword', - to_file = 'tootrss_usercred.txt' - ) - ``` - - copy/paste file names in **config.yml.example** ('_client_id_file_' and '_access_token_file_') + - for **Mastodon** : see [Python wrapper for the Mastodon API](https://mastodonpy.readthedocs.io/) + - Generate the client and user credentials manually via the [Mastodon client](https://mastodonpy.readthedocs.io/en/latest/#app-registration-and-user-authentication) + - the file names for **client_id** and **access_token_file** go in the mastodon section of **config.yml** + - Or use the included script (`python3 create_mastodon_client.py`) which will register your app and prompt you to log in, creating the credential files for you. -Rename the config file **config.yml**. - Start the server ```bash @@ -71,7 +56,7 @@ $ python3 -m flask run --host=0.0.0.0 - the RSS feeds are available on these urls : - for Twitter : http://localhost:5000/_keywords_ or http://localhost:5000/tweets/_keywords_ - - for Mastodon : http://localhost:5000/toots/_keywords_ + - for Mastodon : http://localhost:5000/toots/_keywords_ and http://localhost:5000/toot_favorites ## Examples : ### Search on Twitter : From 20e8aba6cd55c888a92293519cae828e17d8884e Mon Sep 17 00:00:00 2001 From: George Dorn Date: Sat, 2 Dec 2017 03:52:47 +0000 Subject: [PATCH 4/7] Update README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 980a540..0add760 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ The RSS feed displays only the original tweets (not the retweets) and : - [Feedgenerator](https://pypi.python.org/pypi/feedgenerator) - [Tweepy](https://github.com/tweepy/tweepy) - [pytz](https://pypi.python.org/pypi/pytz/) +- [PyYAML](https://github.com/yaml/pyyaml) - [BeautifulSoup](https://pypi.python.org/pypi/beautifulsoup4) - [Mastodon.py](https://github.com/halcy/Mastodon.py) - API keys Twitter and/or Mastodon @@ -44,9 +45,9 @@ $ git clone https://github.com/SamR1/python-twootfeed.git copy/paste the Twitter API key values in **config.yml.example** ('_consumerKey_' and '_consumerSecret_') - for **Mastodon** : see [Python wrapper for the Mastodon API](https://mastodonpy.readthedocs.io/) - Generate the client and user credentials manually via the [Mastodon client](https://mastodonpy.readthedocs.io/en/latest/#app-registration-and-user-authentication) + - note that using an instance other than https://mastodon.social requires adding `api_base_url` to most method calls. - the file names for **client_id** and **access_token_file** go in the mastodon section of **config.yml** - Or use the included script (`python3 create_mastodon_client.py`) which will register your app and prompt you to log in, creating the credential files for you. - - Start the server ```bash From 433e35accfcf014927dfb710be03065c6ab219e4 Mon Sep 17 00:00:00 2001 From: "@gdorn@social.coop" Date: Fri, 1 Dec 2017 20:06:11 -0800 Subject: [PATCH 5/7] Cleanup --- app.py | 1 - create_mastodon_client.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/app.py b/app.py index 6ebff75..bf9d8fb 100755 --- a/app.py +++ b/app.py @@ -235,7 +235,6 @@ def toot_favorites_feed(): if mastodonOK: buffered = [] -# hashtagResult = mastodon.timeline_hashtag(query_feed) favorite_toots = mastodon.favourites() for toot in favorite_toots: diff --git a/create_mastodon_client.py b/create_mastodon_client.py index cc48d03..0733a76 100755 --- a/create_mastodon_client.py +++ b/create_mastodon_client.py @@ -9,7 +9,7 @@ mast_cfg = config['mastodon'] print("Configuration found.") - print("Looks like you want to use this instance:", mast_cfg['url']) + print("Looks like you want to use this instance: ", mast_cfg['url']) print("If that's wrong, now is a good time to cancel (^C) and fix it.") input(" to continue") From 961d9a6106706bc62c02d848b59eba2086e3d1de Mon Sep 17 00:00:00 2001 From: "@gdorn@social.coop" Date: Fri, 1 Dec 2017 21:39:08 -0800 Subject: [PATCH 6/7] Set encoding for config file during read --- config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.py b/config.py index bd67e20..f1e93b9 100755 --- a/config.py +++ b/config.py @@ -4,7 +4,7 @@ import yaml def get_config(): - with open('config.yml', 'r') as stream: + with open('config.yml', 'r', encoding='utf-8') as stream: try: return yaml.safe_load(stream) except yaml.YAMLError as e: From cb3b907189394adaedf07acdc246da87e89f2dd5 Mon Sep 17 00:00:00 2001 From: "@gdorn@social.coop" Date: Fri, 1 Dec 2017 21:53:21 -0800 Subject: [PATCH 7/7] Make text length limit configurable --- app.py | 11 ++++++----- config.example.yml | 1 + 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/app.py b/app.py index bf9d8fb..e3149a3 100755 --- a/app.py +++ b/app.py @@ -19,6 +19,8 @@ param = get_config() +text_length_limit = int(param['feed'].get('text_length_limit', 100)) + # Twitter try: consumerKey = param['twitter']['consumerKey'] @@ -43,7 +45,6 @@ raise Exception("File not found: " + client_file) mastodon_url = param['mastodon'].get('url', 'https://mastodon.social') - print("Using Mastodon instance at:", mastodon_url) mastodon = Mastodon( client_id=client_file, @@ -215,8 +216,8 @@ def tootfeed(query_feed): if not pubdate.tzinfo: pubdate = utc.localize(pubdate).astimezone(pytz.timezone(param['feed']['timezone'])) - if len(text) > 100: - text = text[:100] + '... ' + if len(text) > text_length_limit: + text = text[:text_length_limit] + '... ' f.add_item(title=toot['account']['display_name'] + ' (' + toot['account']['username'] + '): ' + text, link=toot['url'], @@ -265,8 +266,8 @@ def toot_favorites_feed(): if not pubdate.tzinfo: pubdate = utc.localize(pubdate).astimezone(pytz.timezone(param['feed']['timezone'])) - if len(text) > 100: - text = text[:100] + '... ' + if len(text) > text_length_limit: + text = text[:text_length_limit] + '... ' f.add_item(title=toot['account']['display_name'] + ' (' + toot['account']['username'] + '): ' + text, link=toot['url'], diff --git a/config.example.yml b/config.example.yml index 248a4f2..b2bccd9 100755 --- a/config.example.yml +++ b/config.example.yml @@ -16,3 +16,4 @@ feed: author_name: '' feed_url: 'http://localhost:5000/' timezone: 'Europe/Paris' + text_length_limit: 100