From c5ce81a61ce2b04562e3be616427efaa627f019f Mon Sep 17 00:00:00 2001 From: root Date: Tue, 14 Jan 2014 08:32:14 -0600 Subject: add mediengewitter.krebsco.de/magnets --- .gitmodules | 3 +++ sites/mediengewitter.krebsco.de/README.md | 11 +++++++++++ sites/mediengewitter.krebsco.de/magnets/Dockerfile | 17 +++++++++++++++++ sites/mediengewitter.krebsco.de/magnets/README.md | 5 +++++ sites/mediengewitter.krebsco.de/magnets/magnets | 1 + 5 files changed, 37 insertions(+) create mode 100644 sites/mediengewitter.krebsco.de/README.md create mode 100644 sites/mediengewitter.krebsco.de/magnets/Dockerfile create mode 100644 sites/mediengewitter.krebsco.de/magnets/README.md create mode 160000 sites/mediengewitter.krebsco.de/magnets/magnets diff --git a/.gitmodules b/.gitmodules index b59b012a..3fcb4693 100644 --- a/.gitmodules +++ b/.gitmodules @@ -34,3 +34,6 @@ [submodule "sites/paste.retiolum/bump"] path = sites/paste.retiolum/bump url = git@github.com:makefu/bump.git +[submodule "sites/mediengewitter.krebsco.de/magnets/magnets"] + path = sites/mediengewitter.krebsco.de/magnets/magnets + url = https://github.com/makefu/magnets.git diff --git a/sites/mediengewitter.krebsco.de/README.md b/sites/mediengewitter.krebsco.de/README.md new file mode 100644 index 00000000..e7bd88a2 --- /dev/null +++ b/sites/mediengewitter.krebsco.de/README.md @@ -0,0 +1,11 @@ +# Mediengewitter +Broadcast funny pictures. + +Mediengewitter is divided into two parts: +1. the image crawler (magnets) +2. the website (mediengewitter) + +Mediengewitter is deployed via docker containerization. + +# magnets + diff --git a/sites/mediengewitter.krebsco.de/magnets/Dockerfile b/sites/mediengewitter.krebsco.de/magnets/Dockerfile new file mode 100644 index 00000000..8bf400d5 --- /dev/null +++ b/sites/mediengewitter.krebsco.de/magnets/Dockerfile @@ -0,0 +1,17 @@ +FROM ubuntu:latest +RUN echo "deb http://archive.ubuntu.com/ubuntu precise main universe" > /etc/apt/sources.list +RUN dpkg-divert --local --rename --add /sbin/initctl +RUN ln -s /bin/true /sbin/initctl +RUN apt-get update +RUN apt-get install -y python-software-properties python g++ make +RUN add-apt-repository -y ppa:chris-lea/node.js +RUN apt-get update +RUN apt-get -y install nodejs +RUN mkdir -p /opt/magnets +Add magnets/ /magnets +RUN cd /magnets && npm install +# fix crappy wwwdude +RUN cp /magnets/node_modules/wwwdude/lib/wwwdude/node-versions/v0.5.x.js /magnets/node_modules/wwwdude/lib/wwwdude/node-versions/v0.10..js +# -v /media/ext/magnet_pics/:/images +ENV image_folder /images +CMD ["/magnets/run.sh"] diff --git a/sites/mediengewitter.krebsco.de/magnets/README.md b/sites/mediengewitter.krebsco.de/magnets/README.md new file mode 100644 index 00000000..08de8823 --- /dev/null +++ b/sites/mediengewitter.krebsco.de/magnets/README.md @@ -0,0 +1,5 @@ +# docker + + docker build -t krebs/magnets . + # autostart this somehow + docker run -v /media/ext/magnet_pics/:/images krebs/magnets diff --git a/sites/mediengewitter.krebsco.de/magnets/magnets b/sites/mediengewitter.krebsco.de/magnets/magnets new file mode 160000 index 00000000..7f8925b2 --- /dev/null +++ b/sites/mediengewitter.krebsco.de/magnets/magnets @@ -0,0 +1 @@ +Subproject commit 7f8925b2779706c381416f4d578385f520ad4624 -- cgit v1.2.3 From cf8ff0d37ad9a5864699814c69a73433661bae54 Mon Sep 17 00:00:00 2001 From: root Date: Tue, 14 Jan 2014 08:41:52 -0600 Subject: magnets README.md -> DEPLOY.md --- .gitmodules | 3 +++ sites/mediengewitter.krebsco.de/magnets/DEPLOY.md | 7 +++++++ sites/mediengewitter.krebsco.de/magnets/README.md | 5 ----- sites/mediengewitter.krebsco.de/mediengewitter/mediengewitter | 1 + 4 files changed, 11 insertions(+), 5 deletions(-) create mode 100644 sites/mediengewitter.krebsco.de/magnets/DEPLOY.md delete mode 100644 sites/mediengewitter.krebsco.de/magnets/README.md create mode 160000 sites/mediengewitter.krebsco.de/mediengewitter/mediengewitter diff --git a/.gitmodules b/.gitmodules index 3fcb4693..ef1a477d 100644 --- a/.gitmodules +++ b/.gitmodules @@ -37,3 +37,6 @@ [submodule "sites/mediengewitter.krebsco.de/magnets/magnets"] path = sites/mediengewitter.krebsco.de/magnets/magnets url = https://github.com/makefu/magnets.git +[submodule "sites/mediengewitter.krebsco.de/mediengewitter/mediengewitter"] + path = sites/mediengewitter.krebsco.de/mediengewitter/mediengewitter + url = https://github.com/makefu/mediengewitter.git diff --git a/sites/mediengewitter.krebsco.de/magnets/DEPLOY.md b/sites/mediengewitter.krebsco.de/magnets/DEPLOY.md new file mode 100644 index 00000000..f422f79f --- /dev/null +++ b/sites/mediengewitter.krebsco.de/magnets/DEPLOY.md @@ -0,0 +1,7 @@ +# docker + +/media/ext/magnet_pics is the path to a lot of disk space which will be shared by magnets and mediengewitter. + + docker build -t krebs/magnets . + # autostart this somehow + docker run -v /media/ext/magnet_pics/:/images krebs/magnets diff --git a/sites/mediengewitter.krebsco.de/magnets/README.md b/sites/mediengewitter.krebsco.de/magnets/README.md deleted file mode 100644 index 08de8823..00000000 --- a/sites/mediengewitter.krebsco.de/magnets/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# docker - - docker build -t krebs/magnets . - # autostart this somehow - docker run -v /media/ext/magnet_pics/:/images krebs/magnets diff --git a/sites/mediengewitter.krebsco.de/mediengewitter/mediengewitter b/sites/mediengewitter.krebsco.de/mediengewitter/mediengewitter new file mode 160000 index 00000000..db557ff9 --- /dev/null +++ b/sites/mediengewitter.krebsco.de/mediengewitter/mediengewitter @@ -0,0 +1 @@ +Subproject commit db557ff99e9e420b8f85241514c211f20e3ca966 -- cgit v1.2.3 From b9d3ac29f93550cb0eebe9f95d3a2d042838abe7 Mon Sep 17 00:00:00 2001 From: root Date: Tue, 14 Jan 2014 08:42:22 -0600 Subject: add mediengewitter --- .../mediengewitter.krebsco.de/mediengewitter/DEPLOY.md | 9 +++++++++ .../mediengewitter.krebsco.de/mediengewitter/Dockerfile | 17 +++++++++++++++++ .../sites-available/mediengewitter.krebsco.de.conf | 15 +++++++++++++++ 3 files changed, 41 insertions(+) create mode 100644 sites/mediengewitter.krebsco.de/mediengewitter/DEPLOY.md create mode 100644 sites/mediengewitter.krebsco.de/mediengewitter/Dockerfile create mode 100644 sites/mediengewitter.krebsco.de/mediengewitter/etc/nginx/sites-available/mediengewitter.krebsco.de.conf diff --git a/sites/mediengewitter.krebsco.de/mediengewitter/DEPLOY.md b/sites/mediengewitter.krebsco.de/mediengewitter/DEPLOY.md new file mode 100644 index 00000000..7cf8826a --- /dev/null +++ b/sites/mediengewitter.krebsco.de/mediengewitter/DEPLOY.md @@ -0,0 +1,9 @@ +# docker + + docker build -t krebs/mediengewitter . + # autostart this somehow + docker run -p 127.0.0.1::8080 -v /media/ext/magnet_pics/:/images krebs/mediengewitter + +# nginx + + cp etc/nginx/sites-available/mediengewitter.krebsco.de.conf /etc/nginx/sites-available/ diff --git a/sites/mediengewitter.krebsco.de/mediengewitter/Dockerfile b/sites/mediengewitter.krebsco.de/mediengewitter/Dockerfile new file mode 100644 index 00000000..dbfb2a8b --- /dev/null +++ b/sites/mediengewitter.krebsco.de/mediengewitter/Dockerfile @@ -0,0 +1,17 @@ +FROM ubuntu:latest +RUN echo "deb http://archive.ubuntu.com/ubuntu precise main universe" > /etc/apt/sources.list +RUN dpkg-divert --local --rename --add /sbin/initctl +RUN ln -s /bin/true /sbin/initctl +RUN apt-get update +RUN apt-get install -y python-software-properties python g++ make +RUN add-apt-repository -y ppa:chris-lea/node.js +RUN apt-get update +RUN apt-get -y install nodejs +Add mediengewitter/ /mediengewitter +RUN cd /mediengewitter && npm install +# -v /media/ext/magnet_pics/:/images +EXPOSE 8080 +ENV PORT 8080 +RUN rm /mediengewitter/public/content -r +RUN ln -s /images /mediengewitter/public/content +CMD ["/mediengewitter/run.sh"] diff --git a/sites/mediengewitter.krebsco.de/mediengewitter/etc/nginx/sites-available/mediengewitter.krebsco.de.conf b/sites/mediengewitter.krebsco.de/mediengewitter/etc/nginx/sites-available/mediengewitter.krebsco.de.conf new file mode 100644 index 00000000..90ab803c --- /dev/null +++ b/sites/mediengewitter.krebsco.de/mediengewitter/etc/nginx/sites-available/mediengewitter.krebsco.de.conf @@ -0,0 +1,15 @@ +server { + listen 80; + server_name mediengewitter.krebsco.de; +# access_log /var/log/nginx/log/mediengewitter.log; +# error_log /var/log/nginx/log/mediengewitter_error.log; + default_type text/plain; + location / { + proxy_pass http://127.0.0.1:8080/; + proxy_redirect off; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } +} + -- cgit v1.2.3 From ec8862ee06a785656d03840ece949e4237e645c5 Mon Sep 17 00:00:00 2001 From: root Date: Tue, 14 Jan 2014 11:03:25 -0500 Subject: gitmodules: ssh -> https for bump --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index ef1a477d..336b225d 100644 --- a/.gitmodules +++ b/.gitmodules @@ -33,7 +33,7 @@ url = https://github.com/krebscode/krebscode.github.com [submodule "sites/paste.retiolum/bump"] path = sites/paste.retiolum/bump - url = git@github.com:makefu/bump.git + url = https://github.com/makefu/bump.git [submodule "sites/mediengewitter.krebsco.de/magnets/magnets"] path = sites/mediengewitter.krebsco.de/magnets/magnets url = https://github.com/makefu/magnets.git -- cgit v1.2.3 From 46ed619c2bd334f64e2d0a9eaa43cf73f5fd5bde Mon Sep 17 00:00:00 2001 From: lassulus Date: Wed, 15 Jan 2014 15:33:00 +0100 Subject: news: control channel now #news --- news/newsbot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/news/newsbot.py b/news/newsbot.py index 5850e4e9..54a2e7d2 100644 --- a/news/newsbot.py +++ b/news/newsbot.py @@ -221,7 +221,7 @@ url_shortener = 'http://wall' init_channels = ['#news'] bots = {} -knews = NewsBot('knews') +knews = NewsBot('knews', init_channels) #config file reading F = open(feedfile, "r") -- cgit v1.2.3 From c04f99d787e56c7eb5db611bcbcef8269e1b3903 Mon Sep 17 00:00:00 2001 From: lassulus Date: Mon, 20 Jan 2014 15:18:52 +0100 Subject: news: removed old sourcefiles --- news/controller.py | 151 ----------------------------------------------------- news/rssbot.py | 116 ---------------------------------------- 2 files changed, 267 deletions(-) delete mode 100755 news/controller.py delete mode 100755 news/rssbot.py diff --git a/news/controller.py b/news/controller.py deleted file mode 100755 index e9e94476..00000000 --- a/news/controller.py +++ /dev/null @@ -1,151 +0,0 @@ -from time import sleep -import irc.bot -import _thread -import rssbot -import os -import subprocess - -class NewsBot(irc.bot.SingleServerIRCBot): - def __init__(self, name, chans=['#news'], server='ire', port=6667, timeout=60): - irc.bot.SingleServerIRCBot.__init__(self, [(server, port)], name, name) - self.name = name - self.server = server - self.port = port - self.chans = chans - self.to = timeout - - def start(self): - self.bot = _thread.start_new_thread(irc.bot.SingleServerIRCBot.start, (self,)) - - def on_welcome(self, connection, event): - for chan in self.chans: - connection.join(chan) - - def send(self, target, string): - for line in string.split('\n'): - self.connection.action(target, line) - sleep(1) - - def on_privmsg(self, connection, event): - args_array = event.arguments[0].split() - answer = self.read_message(args_array) - self.send(event.source.nick, answer) - - def on_pubmsg(self, connection, event): - args_array = event.arguments[0].split() - if len(args_array) > 0 and args_array[0][:-1]==self.name: - answer = self.read_message(args_array[1:]) - self.send(event.target, answer) - - def on_invite(self, connection, event): - for chan in event.arguments: - connection.join(chan) - - def read_message(self, args): - try: - if args[0] in [x for x in commands.__dict__.keys() if x.find('_')]: - func = getattr(commands, args[0]) - return func(args) - else: - return 'command not found' - except: - return "mimimimi" - - - -class commands(): - def add(args): - if args[1] not in bots and not args[1]==knews.name: - bot = rssbot.RssBot(args[2], args[1], url_shortener=url_shortener) - bots[args[1]] = bot - bot.start() - return "bot " + args[1] + " added" - else: - return args[1] + ' does already exist' - - def delete(args): - bots[args[1]].stop() - del bots[args[1]] - return "bot " + args[1] + " deleted" - - def rename(args): - if args[1] in bots: - if args[2] in bots: - return args[2] + ' already taken' - else: - bots[args[1]].connection.nick(args[2]) - bots[args[1]].name = args[2] - bots[args[2]] = bots[args[1]] - del bots[args[1]] - return 'renamed ' + args[1] + ' in ' + args[2] - else: - return args[1] + ' does not exist' - - def save(args): - output_buffer = '' - for bot in bots: - if bots[bot].loop: - output_buffer += bot + '|' + bots[bot].url + '|' + ' '.join(bots[bot].channels) + '\n' - - F = open(feedfile, "w") - F.writelines(output_buffer) - F.close() - - return "bots saved to " + feedfile - - def caps(args): - return ' '.join([x for x in commands.__dict__.keys() if x.find('_')]) - - def list(args): - output_buffer = '' - for bot in bots: - output_buffer += bot + ' url: ' + bots[bot].url + '\n' - return output_buffer - - def info(args): - if args[1] in bots: - output_buffer = '' - for data in ['title', 'link', 'updated']: - if data in bots[args[1]].feed.feed: - output_buffer += data + ': ' + bots[args[1]].feed.feed[data] + '\n' - output_buffer += 'lastnew: ' + bots[args[1]].lastnew.isoformat() + '\n' - output_buffer += 'rssurl: ' + bots[args[1]].url - return output_buffer - else: - return 'bot not found' - - def search(args): - output = subprocess.check_output(['./GfindFeeds4bot', args[1]]).decode() - return output - - def uptime(args): - output = subprocess.check_output(['uptime']).decode() - return output - -feedfile = 'new_feeds' -url_shortener = 'http://wall' -init_channels = ['#news'] - -if 'FEEDFILE' in os.environ: - feedfile = os.environ['FEEDFILE'] - -if 'URLSHORT' in os.environ: - url_shortener = os.environ['URLSHORT'] - -bots = {} -knews = NewsBot('knews') - -#config file reading -F = open(feedfile, "r") -lines = F.readlines() -F.close() - -for line in lines: - line = line.strip('\n') - linear = line.split('|') - bot = rssbot.RssBot(linear[1], linear[0], init_channels + linear[2].split(), url_shortener) - bot.start() - bots[linear[0]] = bot - -knews.start() - diff --git a/news/rssbot.py b/news/rssbot.py deleted file mode 100755 index 87c58781..00000000 --- a/news/rssbot.py +++ /dev/null @@ -1,116 +0,0 @@ -#!/usr/bin/python -import irc.bot -from irc.client import IRC -import feedparser -import threading -import math -import re -import subprocess -from datetime import datetime -from time import sleep - -class RssBot(irc.bot.SingleServerIRCBot): - def __init__(self, rss, name, chans=['#news'], url_shortener="http://localhost", server='ire', port=6667, timeout=60): - irc.bot.SingleServerIRCBot.__init__(self, [(server, port)], name, name) - self.url = rss - self.name = name - self.server = server - self.port = port - self.chans = chans - self.to = timeout - self.oldnews = [] - self.sendqueue = [] - self.loop = True - self.lastnew = datetime.now() - self.url_shortener = url_shortener - - def better_loop(timeout=0.2): - while self.loop: - self.ircobj.process_once(timeout) - self.ircobj.process_forever = better_loop - - - def start(self): - self.upd_loop = threading.Thread(target=self.updateloop) - self.bot = threading.Thread(target=irc.bot.SingleServerIRCBot.start, args=(self,)) - self.upd_loop.start() - self.bot.start() - - def stop(self): - self.ircobj.disconnect_all() - self.loop = False - del self - - def updateloop(self): - failcount=0 - while True: - try: - self.feed = feedparser.parse(self.url) - for entry in self.feed.entries: - self.oldnews.append(entry.link) - break - except: - print(self.name + ': rss timeout occured') - failcount+=1 - if failcount>20: - print(self.name + ' is broken, going to die') - self.stop() - return - while self.loop: - try: - self.feed = feedparser.parse(self.url) - for entry in self.feed.entries: - if not entry.link in self.oldnews: - #try: - # self.send(entry.title + " " + entry.link + " com: " + entry.comments) - #except AttributeError: - shorturl = self.shortenurl(entry.link) - self.sendall(entry.title + ' ' + shorturl) - self.oldnews.append(entry.link) - self.lastnew = datetime.now() - except: - print(self.name + ': rss timeout occured') - sleep(self.to) - - def shortenurl(self, url): - while True: - try: - shorturl = subprocess.check_output(["curl", "-sS", "-F", "uri=" + url, self.url_shortener]).decode().strip('\n').strip('\r') + '#' + url.partition('://')[2].partition('/')[0] - return shorturl - except: - print('url shortener error') - sleep(1) - - def last(self, target, num): - for feed in [x for x in self.feed.entries][:num]: - self.send(target, feed.title + ' ' + self.shortenurl(feed.link)) - - def sendall(self, string): - for chan in self.channels: - self.send(chan, string) - - def send(self, target, string): - if self.connection.connected: - for line in string.split('\n'): - if len(line) < 450: - self.connection.privmsg(target, line) - sleep(1) - else: - space = 0 - for x in range(math.ceil(len(line)/400)): - oldspace = space - space = line.find(" ", (x+1)*400, (x+1)*400+50) - self.connection.privmsg(target, line[oldspace:space]) - sleep(1) - else: - self.connection.reconnect() - sleep(1) - self.send(string) - - def on_invite(self, connection, event): - for chan in event.arguments: - connection.join(chan) - - def on_welcome(self, connection, event): - for chan in self.chans: - connection.join(chan) -- cgit v1.2.3