diff options
author | lassulus <lass@blue.r> | 2018-07-06 17:42:04 +0200 |
---|---|---|
committer | lassulus <lass@blue.r> | 2018-07-06 17:44:32 +0200 |
commit | 92aa5bb2329c39cf97d4399839989e7401820ae4 (patch) | |
tree | 896a5e81402ebe4e9e4cda083c6b4988b845b691 /krebs/5pkgs/simple/Reaktor/plugins.nix | |
parent | 70e8c4b0a4255a989bf93fe6fca96244ee617eee (diff) |
Reaktor url-title: fix some issues with weird urls
ref: https://irc-bot-science.clsr.net/
Diffstat (limited to 'krebs/5pkgs/simple/Reaktor/plugins.nix')
-rw-r--r-- | krebs/5pkgs/simple/Reaktor/plugins.nix | 24 |
1 files changed, 15 insertions, 9 deletions
diff --git a/krebs/5pkgs/simple/Reaktor/plugins.nix b/krebs/5pkgs/simple/Reaktor/plugins.nix index cd389366e..4a7917b68 100644 --- a/krebs/5pkgs/simple/Reaktor/plugins.nix +++ b/krebs/5pkgs/simple/Reaktor/plugins.nix @@ -121,21 +121,27 @@ rec { pattern = "^.*(?P<args>http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+).*$$"; path = with pkgs; [ curl perl ]; script = pkgs.writePython3 "url-title" [ "beautifulsoup4" "lxml" ] '' + import cgi import sys import urllib.request from bs4 import BeautifulSoup try: - soup = BeautifulSoup(urllib.request.urlopen(sys.argv[1]), "lxml") - title = soup.find('title').string + resp = urllib.request.urlopen(sys.argv[1]) + if resp.headers['content-type'].find('text/html') >= 0: + soup = BeautifulSoup(resp.read(16000), "lxml") + title = soup.find('title').string - if title: - if len(title) > 512: - print('message to long, skipped') - elif len(title.split('\n')) > 5: - print('to many lines, skipped') - else: - print(title) + if title: + if len(title) > 450: + print('message to long, rest skipped') + elif len(title.split('\n')) > 5: + print('to many lines, skipped') + else: + print(title) + else: + cd_header = resp.headers['content-disposition'] + print(cgi.parse_header(cd_header)[1]['filename']) except: # noqa: E722 pass ''; |