Merge branch 'master' of ssh://github.com/krebscode/painload

6 files changed, 176 insertions, 93 deletions

diff --git a/ircbot/new_feeds b/ircbot/new_feeds
deleted file mode 100644
index b1e77657..00000000
--- a/ircbot/new_feeds
+++ /dev/null
@@ -1,56 +0,0 @@
-spiegel_top|http://www.spiegel.de/schlagzeilen/tops/index.rss|#news
-wsj_jp|http://blogs.wsj.com/japanrealtime/feed?mod=WSJ_Japan_JapanRealTime|#news
-ccc|http://www.ccc.de/rss/updates.rdf|#news
-coinspotting|http://coinspotting.com/rss|#news
-reddit_sci|http://www.reddit.com/r/science/.rss|#news
-reddit_tech|http://www.reddit.com/r/technology/.rss|#news
-scmp|http://www.scmp.com/rss/91/feed|#news
-golem|http://www.golem.de/rss.php?feed=RSS1.0|#news
-anon|http://anoninsiders.net/feed/|#news
-wsj_kor|http://blogs.wsj.com/korearealtime/feed?mod=WSJ_korearealtimeRealTime|#news #test
-faz_politik|http://www.faz.net/rss/aktuell/politik/|#news
-reddit_prog|http://reddit.com/r/programming/|#news
-gulli|http://ticker.gulli.com/rss/|#news
-danisch|http://www.danisch.de/blog/feed/|#news
-lisp|http://planet.lisp.org/rss20.xml|#news
-wsj_in|http://blogs.wsj.com/indiarealtime/feed?mod=irt|#news #test
-wsj_me|http://blogs.wsj.com/middleeast/feed?mod=middleeast|#news #test
-lolmythesis|http://lolmythesis.com/rss|#news
-exploitdb|http://www.exploit-db.com/rss.xml|#news
-LtU|http://lambda-the-ultimate.org/rss.xml|#news
-HN|http://news.ycombinator.com/rss|#news
-weechat|http://dev.weechat.org/feed/atom|#news
-faz_feui|http://www.faz.net/rss/aktuell/feuilleton/|#news
-fvwm|http://freecode.com/projects/fvwm/releases.atom|#news
-rawstory|http://www.rawstory.com/rs/feed/|#news
-wsj_cn|http://blogs.wsj.com/chinarealtime/feed?mod=chinablog|#news
-torr_news|http://feed.torrentfreak.com/Torrentfreak/|#news
-xkcd|https://xkcd.com/rss.xml|#news
-wp_world|http://feeds.washingtonpost.com/rss/rss_blogpost|#news
-wsj_eu|http://blogs.wsj.com/emergingeurope/feed?mod=emergingeurope|#news #test
-telepolis|http://www.heise.de/tp/rss/news-atom.xml|#news
-wsj_sea|http://blogs.wsj.com/searealtime/feed?mod=WSJ_SEA_Blog|#news #test
-embargowatch|https://embargowatch.wordpress.com/feed/|#news
-aje|http://www.aljazeera.com/Services/Rss/?PostingId=2007731105943979989|#news
-taz|http://taz.de/Themen-des-Tages/!p15;rss/|#news
-reuters|http://feeds.reuters.com/Reuters/worldNews|#news
-sec-db|http://feeds.security-database.com/SecurityDatabaseToolsWatch|#news
-archlinux|http://www.archlinux.org/feeds/news/|#news
-nnewsg|http://www.net-news-global.net/rss/rssfeed.xml|#news
-faz_wirtschaft|http://www.faz.net/rss/aktuell/wirtschaft/|#news
-arbor|http://feeds2.feedburner.com/asert/|#news
-reddit_world|http://www.reddit.com/r/worldnews/.rss|#news
-linuxinsider|http://www.linuxinsider.com/perl/syndication/rssfull.pl|#news
-spiegel_eil|http://www.spiegel.de/schlagzeilen/eilmeldungen/index.rss|#news
-heise|http://heise.de.feedsportal.com/c/35207/f/653902/index.rss|#news
-slashdot|http://rss.slashdot.org/Slashdot/slashdot|#news
-antirez|http://antirez.com/rss|#news
-sz|http://suche.sueddeutsche.de/?output=rss|#news
-GerForPol|http://www.german-foreign-policy.com/de/news/rss-2.0|#news
-schallurauch|http://feeds.feedburner.com/SchallUndRauch|#news
-torr_bits|http://feeds.feedburner.com/TorrentfreakBits|#news
-fefe|http://blog.fefe.de/rss.xml|#news
-rt|http://rt.com/rss/news/|#news
-ars|http://feeds.arstechnica.com/arstechnica/index?format=xml|#news
-dwn|http://deutsche-wirtschafts-nachrichten.de/feed/customfeed/|#news
-reddit_consp|http://reddit.com/r/conspiracy/.rss|#news
diff --git a/ircbot/GfindFeeds4bot b/news/GfindFeeds4bot
index a5439c5a..a5439c5a 100755
--- a/ircbot/GfindFeeds4bot
+++ b/news/GfindFeeds4bot
diff --git a/ircbot/contoller.py b/news/controller.py
index 1d9347c3..e9e94476 100755
--- a/ircbot/contoller.py
+++ b/news/controller.py
@@ -3,8 +3,7 @@ import irc.bot
 import _thread
 import rssbot
 import os
-
-
+import subprocess
 
 class NewsBot(irc.bot.SingleServerIRCBot):
     def __init__(self, name, chans=['#news'], server='ire', port=6667, timeout=60):
@@ -24,22 +23,17 @@ class NewsBot(irc.bot.SingleServerIRCBot):
 
     def send(self, target, string):
         for line in string.split('\n'):
-            self.connection.privmsg(target, line)
-            sleep(1)
-
-    def sendq(self, target, string):
-        for line in string.split('\n'):
-            self.connection.privmsg(target, line)
+            self.connection.action(target, line)
             sleep(1)
 
     def on_privmsg(self, connection, event):
         args_array = event.arguments[0].split()
         answer = self.read_message(args_array)
-        self.sendq(event.source.nick, answer)
+        self.send(event.source.nick, answer)
 
     def on_pubmsg(self, connection, event):
         args_array = event.arguments[0].split()
-        if args_array[0][:-1]==self.name:
+        if len(args_array) > 0 and args_array[0][:-1]==self.name:
             answer = self.read_message(args_array[1:])
             self.send(event.target, answer)
 
@@ -61,10 +55,13 @@ class NewsBot(irc.bot.SingleServerIRCBot):
 
 class commands():
     def add(args): 
-        bot = rssbot.RssBot(args[2], args[1], url_shortener=url_shortener)
-        bots[args[1]] = bot
-        bot.start()
-        return "bot " + args[1] + " added"
+        if args[1] not in bots and not args[1]==knews.name:
+            bot = rssbot.RssBot(args[2], args[1], url_shortener=url_shortener)
+            bots[args[1]] = bot
+            bot.start()
+            return "bot " + args[1] + " added"
+        else:
+            return args[1] + ' does already exist'
 
     def delete(args):
         bots[args[1]].stop()
@@ -87,7 +84,8 @@ class commands():
     def save(args):
         output_buffer = ''
         for bot in bots:
-           output_buffer += bot + '|' + bots[bot].url + '|' + ' '.join(bots[bot].channels) + '\n'
+            if bots[bot].loop:
+                output_buffer += bot + '|' + bots[bot].url + '|' + ' '.join(bots[bot].channels) + '\n'
 
         F = open(feedfile, "w")
         F.writelines(output_buffer)
@@ -110,12 +108,19 @@ class commands():
             for data in ['title', 'link', 'updated']:
                 if data in bots[args[1]].feed.feed:
                     output_buffer += data + ': ' + bots[args[1]].feed.feed[data] + '\n'
-            output_buffer += 'lastnew: ' + bots[args[1]].lastnew.isoformat()
+            output_buffer += 'lastnew: ' + bots[args[1]].lastnew.isoformat() + '\n'
+            output_buffer += 'rssurl: ' + bots[args[1]].url
             return output_buffer
         else:
             return 'bot not found'
 
+    def search(args):
+        output = subprocess.check_output(['./GfindFeeds4bot', args[1]]).decode()
+        return output
 
+    def uptime(args):
+        output = subprocess.check_output(['uptime']).decode()
+        return output
 
 feedfile = 'new_feeds'
 url_shortener = 'http://wall'
diff --git a/ircbot/feeds b/news/feeds
index 50fe0667..50fe0667 100644
--- a/ircbot/feeds
+++ b/news/feeds
diff --git a/news/new_feeds b/news/new_feeds
new file mode 100644
index 00000000..97f2f762
--- /dev/null
+++ b/news/new_feeds
@@ -0,0 +1,110 @@
+faz_feui|http://www.faz.net/rss/aktuell/feuilleton/|#news
+catholic_news|http://feeds.feedburner.com/catholicnewsagency/dailynews|#news
+lisp|http://planet.lisp.org/rss20.xml|#news
+sciencemag|http://news.sciencemag.org/rss/current.xml|#news
+weechat|http://dev.weechat.org/feed/atom|#news
+nnewsg|http://www.net-news-global.net/rss/rssfeed.xml|#news
+ccc|http://www.ccc.de/rss/updates.rdf|#news
+danisch|http://www.danisch.de/blog/feed/|#news
+tinc|http://tinc-vpn.org/news/index.rss|#news
+ft_india|http://www.ft.com/rss/home/india|#news
+nasa_news|http://www.nasa-usa.de/rss/dyn/breaking_news.rss|#news
+GerForPol|http://www.german-foreign-policy.com/de/news/rss-2.0|#news
+un_me|http://www.un.org/apps/news/rss/rss_mideast.asp|#news
+shz_news|http://www.shz.de/nachrichten/newsticker/rss|#news
+wp_world|http://feeds.washingtonpost.com/rss/rss_blogpost|#news
+fbi_nat_press|http://www.fbi.gov/news/rss|#news
+dwn|http://deutsche-wirtschafts-nachrichten.de/feed/customfeed/|#news
+faz_politik|http://www.faz.net/rss/aktuell/politik/|#news
+spiegel_top|http://www.spiegel.de/schlagzeilen/tops/index.rss|#news
+presseportal|http://www.presseportal.de/rss/presseportal.rss2|#news
+telegraph_uk|http://www.telegraph.co.uk/news/uknews/rss|#news
+LtU|http://lambda-the-ultimate.org/rss.xml|#news
+vimperator|https://sites.google.com/a/vimperator.org/www/blog/posts.xml|#news
+schallurauch|http://feeds.feedburner.com/SchallUndRauch|#news
+add|kernel|#news
+anon|http://anoninsiders.net/feed/|#news
+4chan_status|http://status.4chan.org/feeds/posts/default?alt=rss|
+nytimes|http://rss.nytimes.com/services/xml/rss/nyt/World.xml|#news
+bbc|http://feeds.bbci.co.uk/news/rss.xml|#news
+bild|http://rss.bild.de/bild.xml|#news
+reddit_world|http://www.reddit.com/r/worldnews/.rss|#news
+fbi|http://www.fbi.gov/homepage/RSS|#news
+reddit_sci|http://www.reddit.com/r/science/.rss|#news
+geheimorganisation|http://geheimorganisation.org/feed/|#news
+eia_press|http://www.eia.gov/rss/press_rss.xml|#news
+nsa|http://www.nsa.gov/rss.shtml|#news
+travel_warnings|http://feeds.travel.state.gov/ca/travelwarnings-alerts|#news
+tigsource|http://www.tigsource.com/feed/|#news
+un_top|http://www.un.org/apps/news/rss/rss_top.asp|#news
+archlinux|http://www.archlinux.org/feeds/news/|#news
+sec-db|http://feeds.security-database.com/SecurityDatabaseToolsWatch|#news
+coinspotting|http://coinspotting.com/rss|#news
+fefe|http://blog.fefe.de/rss.xml|#news
+embargowatch|https://embargowatch.wordpress.com/feed/|#news
+spiegel_eil|http://www.spiegel.de/schlagzeilen/eilmeldungen/index.rss|#news
+aje|http://www.aljazeera.com/Services/Rss/?PostingId=2007731105943979989|#news
+gulli|http://ticker.gulli.com/rss/|#news
+us_math_society|http://www.ams.org/cgi-bin/content/news_items.cgi?rss=1|#news
+tagesschau|http://www.tagesschau.de/newsticker.rdf|#news
+fbi_news|http://www.fbi.gov/news/news_blog/rss.xml|#news
+bmj|http://www.bmj.com/rss|#news
+ft_me|http://www.ft.com/rss/home/middleeast|#news
+fbi_stories|http://www.fbi.gov/news/stories/all-stories/rss.xml|#news
+sz_wirtschaft|http://rss.sueddeutsche.de/rss/Wirtschaft|#news
+arbor|http://feeds2.feedburner.com/asert/|#news
+reddit_tech|http://www.reddit.com/r/technology/.rss|#news
+golem|http://www.golem.de/rss.php?feed=RSS1.0|#news
+heise|http://heise.de.feedsportal.com/c/35207/f/653902/index.rss|#news
+fbi_press|http://www.fbi.gov/news/current/rss.xml|#news
+ars|http://feeds.arstechnica.com/arstechnica/index?format=xml|#news
+cancer|http://feeds.feedburner.com/ncinewsreleases?format=xml|#news
+un_eu|http://www.un.org/apps/news/rss/rss_europe.asp|#news
+europa_ric|http://ec.europa.eu/research/infocentre/rss/infocentre-rss.xml|#news
+fedreserve|http://www.federalreserve.gov/feeds/press_all.xml|#news
+exploitdb|http://www.exploit-db.com/rss.xml|#news
+xkcd|https://xkcd.com/rss.xml|#news
+reddit_prog|http://reddit.com/r/programming/|#news
+HN|http://news.ycombinator.com/rss|#news
+sz_wissen|http://suche.sueddeutsche.de/rss/Wissen|#news
+scmp|http://www.scmp.com/rss/91/feed|#news
+shackspace|http://shackspace.de/?feed=rss2|#news
+greenpeace|http://www.greenpeace.de/nachrichten/feed/rss2/|#news
+rt|http://rt.com/rss/news/|#news
+nasa_iotd|http://www.nasa-usa.de/rss/dyn/lg_image_of_the_day.rss|#news
+z0r|https://www.facebook.com/feeds/page.php?format=atom10&id=278857186139|#news
+stz|http://www.stuttgarter-zeitung.de/rss/topthemen.rss.feed|#news
+reuters|http://feeds.reuters.com/Reuters/worldNews|#news
+gmanet|http://www.gmanetwork.com/news/rss/news|#news
+un_am|http://www.un.org/apps/news/rss/rss_americas.asp|#news
+slashdot|http://rss.slashdot.org/Slashdot/slashdot|#news
+antirez|http://antirez.com/rss|#news
+telegraph_world|http://www.telegraph.co.uk/news/worldnews/rss|#news
+fvwm|http://freecode.com/projects/fvwm/releases.atom|#news
+eu_survei|http://www.eurosurveillance.org/public/RSSFeed/RSS.aspx|#news
+eia_today|http://www.eia.gov/rss/todayinenergy.xml|#news
+reddit_consp|http://reddit.com/r/conspiracy/.rss|#news
+ft_uk|http://www.ft.com/rss/home/uk|#news
+times|http://www.thetimes.co.uk/tto/news/rss|#news
+phys|http://phys.org/rss-feed/|#news
+stern|http://www.stern.de/feed/standard/all/|#news
+zdnet|http://www.zdnet.com/news/rss.xml|#news
+presse_polizei|http://www.presseportal.de/rss/polizei.rss2|#news
+torr_news|http://feed.torrentfreak.com/Torrentfreak/|#news
+faz_wirtschaft|http://www.faz.net/rss/aktuell/wirtschaft/|#news
+telegraph_finance|http://www.telegraph.co.uk/finance/rss|#news
+linuxinsider|http://www.linuxinsider.com/perl/syndication/rssfull.pl|#news
+telegraph_pol|http://www.telegraph.co.uk/news/politics/rss|#news
+lolmythesis|http://lolmythesis.com/rss|#news
+taz|http://taz.de/Themen-des-Tages/!p15;rss/|#news
+un_afr|http://www.un.org/apps/news/rss/rss_africa.asp|#news
+ft_us|http://www.ft.com/rss/home/us|#news
+telepolis|http://www.heise.de/tp/rss/news-atom.xml|#news
+ft_europe|http://www.ft.com/rss/home/europe|#news
+handelblatt|http://www.handelsblatt.com/contentexport/feed/schlagzeilen|#news
+rawstory|http://www.rawstory.com/rs/feed/|#news
+sz_politik|http://rss.sueddeutsche.de/rss/Politik|#news
+un_pac|http://www.un.org/apps/news/rss/rss_asiapac.asp|#news
+torr_bits|http://feeds.feedburner.com/TorrentfreakBits|#news
+ign|http://feeds.ign.com/ign/all|#news
+ft_asia|http://www.ft.com/rss/home/asia|#news
diff --git a/ircbot/rssbot.py b/news/rssbot.py
index 225e6f13..87c58781 100755
--- a/ircbot/rssbot.py
+++ b/news/rssbot.py
@@ -39,45 +39,69 @@ class RssBot(irc.bot.SingleServerIRCBot):
     def stop(self):
         self.ircobj.disconnect_all()
         self.loop = False
+        del self
 
     def updateloop(self):
-        try:
-            self.feed = feedparser.parse(self.url)
-        except:
-            print(self.name + ': rss timeout occured')
-        for entry in self.feed.entries:
-            self.oldnews.append(entry.link)
+        failcount=0
+        while True:
+          try:
+              self.feed = feedparser.parse(self.url)
+              for entry in self.feed.entries:
+                  self.oldnews.append(entry.link)
+              break
+          except:
+              print(self.name + ': rss timeout occured')
+              failcount+=1
+              if failcount>20:
+                  print(self.name + ' is broken, going to die')
+                  self.stop()
+                  return
         while self.loop:
             try:
                 self.feed = feedparser.parse(self.url)
+                for entry in self.feed.entries:
+                    if not entry.link in self.oldnews:
+                        #try:
+                        #    self.send(entry.title + " " + entry.link + " com: " + entry.comments)
+                        #except AttributeError:
+                        shorturl = self.shortenurl(entry.link)
+                        self.sendall(entry.title + ' ' + shorturl)
+                        self.oldnews.append(entry.link)
+                        self.lastnew = datetime.now()
             except:
                 print(self.name + ': rss timeout occured')
-            for entry in self.feed.entries:
-                if not entry.link in self.oldnews:
-                    #try:
-                    #    self.send(entry.title + " " + entry.link + " com: " + entry.comments)
-                    #except AttributeError:
-                    shorturl = subprocess.check_output(["curl", "-sS", "-F", "uri=" + entry.link, self.url_shortener]).decode()
-                    self.send(entry.title + " " + shorturl.strip('\n').strip('\r') + '#' + entry.link.partition('://')[2].partition('/')[0])
-                    self.oldnews.append(entry.link)
-                    self.lastnew = datetime.now()
             sleep(self.to)
 
-    def send(self, string):
+    def shortenurl(self, url):
+      while True:
+          try:
+              shorturl = subprocess.check_output(["curl", "-sS", "-F", "uri=" + url, self.url_shortener]).decode().strip('\n').strip('\r') + '#' + url.partition('://')[2].partition('/')[0]
+              return shorturl
+          except:
+              print('url shortener error')
+              sleep(1)
+
+    def last(self, target, num):
+        for feed in [x for x in self.feed.entries][:num]:
+            self.send(target, feed.title + ' ' + self.shortenurl(feed.link))
+
+    def sendall(self, string):
+        for chan in self.channels:
+            self.send(chan, string)
+
+    def send(self, target, string):
         if self.connection.connected:
             for line in string.split('\n'):
                 if len(line) < 450:
-                    for chan in self.channels:
-                        self.connection.privmsg(chan, line)
+                    self.connection.privmsg(target, line)
+                    sleep(1)
                 else:
                     space = 0
                     for x in range(math.ceil(len(line)/400)):
                         oldspace = space
                         space = line.find(" ", (x+1)*400, (x+1)*400+50)
-                        for chan in self.channels:
-                            self.connection.privmsg(chan, line[oldspace:space])
+                        self.connection.privmsg(target, line[oldspace:space])
                         sleep(1)
-                sleep(1)
         else:
             self.connection.reconnect()
             sleep(1)