From 2a9e511f8dae8e708b2c31863385f91b7965c582 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20LASSERRE?= Date: Mon, 4 Aug 2014 21:22:33 +0200 Subject: [PATCH 01/82] Remove trailing whitespaces. Reorder each files. --- example/deefuzzer.json | 77 ++++++++++++----------- example/deefuzzer.xml | 78 +++++++++++------------ example/deefuzzer.yaml | 48 +++++++------- example/deefuzzer_doc.xml | 129 +++++++++++++++++++------------------- 4 files changed, 167 insertions(+), 165 deletions(-) diff --git a/example/deefuzzer.json b/example/deefuzzer.json index 333f112..fd4acfe 100644 --- a/example/deefuzzer.json +++ b/example/deefuzzer.json @@ -1,60 +1,61 @@ { "deefuzzer": { - "log": "mystation.log", - "m3u": "mystation.m3u", + "log": "/path/to/station.log", + "m3u": "/path/to/station.m3u", "station": { "control": { - "mode": 0, + "mode": 0, "port": 16001 - }, + }, "infos": { - "description": "my_station", - "genre": "music", - "name": "my_station", + "description": "My personal best funky playlist ever !", + "genre": "Various Funk Groove", + "name": "My best funky station", + "short_name": "My_station", "url": "http://parisson.com" - }, + }, "jingles": { - "dir": "/path/to/jingles", - "mode": 0, + "dir": "/path/to/jingles", + "mode": 0, "shuffle": 1 - }, + }, "media": { - "bitrate": 96, - "dir": "/path/to/mp3/", - "format": "mp3", - "m3u": "/path/to/m3u_file", - "ogg_quality": 4, - "samplerate": 48000, - "shuffle": 0, + "bitrate": 96, + "dir": "/path/to/mp3/", + "format": "mp3", + "m3u": "/path/to/m3u_file", + "ogg_quality": 4, + "samplerate": 48000, + "shuffle": 0, "voices": "2" - }, + }, "record": { - "dir": "/home/telecaster/archives/mp3", + "dir": "/path/to/archives", "mode": 0 - }, + }, "relay": { - "author": "Inconnu", - "mode": 0, + "author": "Unknown", + "mode": 0, "url": "http://127.0.0.1:8000/telecaster_live.mp3" - }, + }, "rss": { - "dir": "/var/www/rss", - "enclosure": 0, - "media_url": "http://localhost/rss/" - }, + "dir": "/path/to/rss", + "enclosure": 1, + "media_url": "http://localhost/media/" + }, "server": { - "host": "127.0.0.1", - "mountpoint": "monitor", - "port": 8000, - "public": 0, - "sourcepassword": "source2parisson", + "host": "127.0.0.1", + "mountpoint": "monitor", + "port": 8000, + "public": 0, + "sourcepassword": "icecast_source_password", "type": "icecast" - }, + }, "twitter": { - "key": "76728330-OjKgbHtn4II86Ad7pNUGEzfNAkGTW5Wvw38qUmLE", - "mode": 0, - "secret": "4egZs1dSM37XVY8zXa016Yueku2fleXF2bx8k25V4", - "tags": "bla bla" + "key": "you access token key", + "mode": 0, + "secret": "your access token secret key", + "tags": "parisson deefuzzer" } } } diff --git a/example/deefuzzer.xml b/example/deefuzzer.xml index 9b46c57..c3f48bf 100644 --- a/example/deefuzzer.xml +++ b/example/deefuzzer.xml @@ -1,61 +1,61 @@ - mystation.log - mystation.m3u - + /path/to/station.log + /path/to/station.m3u + + 0 + 16001 + - my_station - My best funky station My personal best funky playlist ever ! - http://mydomain.com Various Funk Groove + My best funky station + My_station + http://parisson.com - - icecast - mydomain.com - 8000 - icecast_source_password - 1 - + + /path/to/jingles + 0 + 1 + + 96 /path/to/mp3/ mp3 - 192 - 7 - 44100 - 2 - 1 /path/to/m3u_file + 4 + 48000 + 0 + 2 + + /path/to/archives + 0 + + + Unknown + 0 + http://127.0.0.1:8000/telecaster_live.mp3 + /path/to/rss/ 1 - http://my.domain.com/media/ + http://localhost/media/ + + 127.0.0.1 + monitor + 8000 + 0 + icecast_source_password + icecast + - 0 your access token key + 0 your access token secret key - bla bla + parisson deefuzzer - - 0 - /path/to/jingles - 1 - - - 0 - 1234 - - - 0 - http://anotherdomain.com:8000/stream.mp3 - Me - - - 0 - /path/to/archives - diff --git a/example/deefuzzer.yaml b/example/deefuzzer.yaml index c2d5712..d663020 100644 --- a/example/deefuzzer.yaml +++ b/example/deefuzzer.yaml @@ -1,47 +1,47 @@ deefuzzer: - log: mystation.log - m3u: mystation.m3u + log: /path/to/station.log + m3u: /path/to/station.m3u station: control: {mode: 0, port: 16001} - server: {host: 127.0.0.1, - mountpoint: monitor, - port: 8000, public: 0, - sourcepassword: source2parisson, - type: icecast} - - infos: {name: my_station, - description: my_station, - genre: music, + infos: {description: 'My personal best funky playlist ever!', + genre: 'Various Funk Groove', + name: 'My best funky station', + short_name: "My_station", url: 'http://parisson.com'} - media: {dir: /path/to/mp3/, - m3u: /path/to/m3u_file, + jingles: {dir: /path/to/jingles, + mode: 0, + shuffle: 1} + + media: {bitrate: 96, + dir: /path/to/mp3/, format: mp3, - bitrate: 96, + m3u: /path/to/m3u_file, ogg_quality: 4, samplerate: 48000, shuffle: 0, voices: '2'} - jingles: {dir: /path/to/jingles, - mode: 0, - shuffle: 1} - - record: {dir: /home/telecaster/archives/mp3, + record: {dir: /path/to/archives, mode: 0} - relay: {author: Inconnu, + relay: {author: Unknown, mode: 0, url: 'http://127.0.0.1:8000/telecaster_live.mp3'} rss: {dir: /var/www/rss, enclosure: 0, - media_url: 'http://localhost/rss/'} + media_url: 'http://localhost/media/'} + server: {host: 127.0.0.1, + mountpoint: monitor, + port: 8000, public: 0, + sourcepassword: icecast_source_password, + type: icecast} - twitter: {key: 76728330-OjKgbHtn4II86Ad7pNUGEzfNAkGTW5Wvw38qUmLE, - mode: 0, secret: 4egZs1dSM37XVY8zXa016Yueku2fleXF2bx8k25V4, - tags: bla bla} + twitter: {key: 'your access token key', + mode: 0, secret: 'your acess token secret key', + tags: 'parisson deefuzzzer'} diff --git a/example/deefuzzer_doc.xml b/example/deefuzzer_doc.xml index 71cff54..aa915be 100644 --- a/example/deefuzzer_doc.xml +++ b/example/deefuzzer_doc.xml @@ -1,55 +1,73 @@ - /path/to/log/mystation.log + /path/to/station.log - /path/to/m3u/mystation.m3u - + /path/to/station.m3u + + + 0 + + 16001 + - - my_station + + My personal best funky playlist ever! My best funky station - - My personal best funky playlist ever ! + + My_station - http://mydomain.com + http://parisson.com Various Funk Groove - - - icecast - - mydomain.com - - 8000 - - icecast_source_password - - 1 - + + + /path/to/jingles + + 0 + + 1 + + + 96 /path/to/mp3/ mp3 - - 192 + + /path/to/m3u_file - 7 + 4 - 44100 + 48000 + + 0 2 - - 1 - - /path/to/m3u_file + + + /path/to/archives + + 0 + + + + Unknown + + 0 + + http://127.0.0.1:8000/telecaster_live.mp3 + - http://my.domain.com/media/ + http://localhost/media/ + + + 127.0.0.1 + + monitor + + 8000 + + 0 + + icecast_source_password + + icecast + - - 0 your access token key + + 0 your access token secret key - bla bla + parisson deefuzzer - - - 0 - - /path/to/jingles - - 1 - - - - 0 - - 1234 - - - - 0 - - http://anotherdomain.com:8000/stream.mp3 - - Me - - - - 0 - - /path/to/archives - From 310bf51dd533611cd33840627918dc12159c7a91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20LASSERRE?= Date: Tue, 5 Aug 2014 19:19:06 +0200 Subject: [PATCH 02/82] Update README.rst Fix github link for DeeFuzzer logo :) --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 3a10da5..5f69cf5 100644 --- a/README.rst +++ b/README.rst @@ -1,4 +1,4 @@ -.. image:: http://github.com/yomguy/DeeFuzzer/raw/master/doc/img/logo_deefuzzer.png +.. image:: https://github.com/yomguy/DeeFuzzer/raw/master/doc/img/logo_deefuzzer.png Introduction ============ From 2146affefefda38a011d02a4c5226a897f5b51c2 Mon Sep 17 00:00:00 2001 From: Guillaume Pellerin Date: Tue, 5 Aug 2014 20:14:36 +0200 Subject: [PATCH 03/82] fix non-absolute log and m3u paths --- deefuzzer/core.py | 14 +++++++++----- deefuzzer/station.py | 2 +- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/deefuzzer/core.py b/deefuzzer/core.py index 3a3f624..a909f93 100644 --- a/deefuzzer/core.py +++ b/deefuzzer/core.py @@ -58,15 +58,19 @@ def __init__(self, conf_file): Thread.__init__(self) self.conf_file = conf_file self.conf = self.get_conf_dict() - + for key in self.conf['deefuzzer'].keys(): if key == 'log': - self.logger = Logger(self.conf['deefuzzer']['log']) + log_file = self.conf['deefuzzer']['log'] + log_dir = os.sep.join(log_file.split(os.sep)[:-1]) + if not os.path.exists(log_dir) and log_dir: + os.makedirs(m3u_dir) + self.logger = Logger(log_file) if key == 'm3u': self.m3u = self.conf['deefuzzer']['m3u'] else: setattr(self, key, self.conf['deefuzzer'][key]) - + if isinstance(self.conf['deefuzzer']['station'], dict): # Fix wrong type data from xmltodict when one station (*) self.nb_stations = 1 @@ -98,7 +102,7 @@ def get_conf_dict(self): def set_m3u_playlist(self): m3u_dir = os.sep.join(self.m3u.split(os.sep)[:-1]) - if not os.path.exists(m3u_dir): + if not os.path.exists(m3u_dir) and m3u_dir: os.makedirs(m3u_dir) m3u = open(self.m3u, 'w') m3u.write('#EXTM3U\n') @@ -123,7 +127,7 @@ def run(self): if self.m3u: self.set_m3u_playlist() - + p = Producer(q) p.start() diff --git a/deefuzzer/station.py b/deefuzzer/station.py index a8935ea..53fe6e9 100644 --- a/deefuzzer/station.py +++ b/deefuzzer/station.py @@ -607,7 +607,7 @@ def ping_server(self): self.logger.write_info(text) self.q.task_done() except: - time.sleep(0.5) + time.sleep(1) if log: text = 'Station ' + self.channel_url + ' : could not connect the channel' self.logger.write_error(text) From f06699d383c41a8fd41907ee38ea4049869281c7 Mon Sep 17 00:00:00 2001 From: Guillaume Pellerin Date: Tue, 5 Aug 2014 20:16:05 +0200 Subject: [PATCH 04/82] fix version --- deefuzzer/__init__.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/deefuzzer/__init__.py b/deefuzzer/__init__.py index ec0697d..e1854b7 100644 --- a/deefuzzer/__init__.py +++ b/deefuzzer/__init__.py @@ -2,4 +2,4 @@ from station import * from tools import * -__version__ = '0.6.5' \ No newline at end of file +__version__ = '0.6.6' \ No newline at end of file diff --git a/setup.py b/setup.py index ce91c8e..0d3c0ec 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,7 @@ long_description = open('README.rst').read(), author = "Guillaume Pellerin", author_email = "yomguy@parisson.com", - version = '0.6.5', + version = '0.6.6', install_requires = [ 'setuptools', 'python-shout', From 74fb0ed05ed6f18ae6abcb9358b5567cb7d7ae88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20LASSERRE?= Date: Wed, 6 Aug 2014 11:34:44 +0200 Subject: [PATCH 05/82] Update song separation mark Replace "artist : title" by "artist - title" --- deefuzzer/station.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/deefuzzer/station.py b/deefuzzer/station.py index a8935ea..0c8fa77 100644 --- a/deefuzzer/station.py +++ b/deefuzzer/station.py @@ -361,7 +361,7 @@ def tweet(self): if not (title or artist): song = str(media_obj.file_name) else: - song = artist + ' : ' + title + song = artist + ' - ' + title song = song.encode('utf-8') artist = artist.encode('utf-8') @@ -485,7 +485,7 @@ def update_rss(self, media_list, rss_file, sub_title): if not (title or artist): song = str(media.file_title) else: - song = artist + ' : ' + title + song = artist + ' - ' + title media_absolute_playtime += media.length @@ -537,7 +537,7 @@ def set_relay_mode(self): self.artist = self.relay_author.encode('utf-8') self.title = self.title.replace('_', ' ') self.artist = self.artist.replace('_', ' ') - self.song = self.artist + ' : ' + self.title + self.song = self.artist + ' - ' + self.title if self.type == 'stream-m': relay = URLReader(self.relay_url) @@ -563,7 +563,7 @@ def set_read_mode(self): if not (self.title or self.artist): song = str(self.current_media_obj[0].file_name) else: - song = self.artist + ' : ' + self.title + song = self.artist + ' - ' + self.title self.song = song.encode('utf-8') self.artist = self.artist.encode('utf-8') From fcd3cba5cc92c12cb0c330a0235f8412cc37a99b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20LASSERRE?= Date: Wed, 6 Aug 2014 11:39:54 +0200 Subject: [PATCH 06/82] Remove channel url in station name --- deefuzzer/station.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deefuzzer/station.py b/deefuzzer/station.py index 0c8fa77..95d0051 100644 --- a/deefuzzer/station.py +++ b/deefuzzer/station.py @@ -116,7 +116,7 @@ def __init__(self, station, q, logger, m3u): sys.exit('Not a compatible server type. Choose "stream-m" or "icecast".') self.channel.url = self.station['infos']['url'] - self.channel.name = self.station['infos']['name'] + ' : ' + self.channel.url + self.channel.name = self.station['infos']['name'] self.channel.genre = self.station['infos']['genre'] self.channel.description = self.station['infos']['description'] self.channel.format = self.media_format From 32e38d4a583f959c0ea2e72d3f76197ffdad6481 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20LASSERRE?= Date: Wed, 6 Aug 2014 11:44:28 +0200 Subject: [PATCH 07/82] Remove 'ogg_quality' in mp3 stream --- deefuzzer/station.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/deefuzzer/station.py b/deefuzzer/station.py index 95d0051..45f6ae2 100644 --- a/deefuzzer/station.py +++ b/deefuzzer/station.py @@ -127,10 +127,15 @@ def __init__(self, station, q, logger, m3u): self.channel.public = int(self.station['server']['public']) self.channel.genre = self.station['infos']['genre'] self.channel.description = self.station['infos']['description'] - self.channel.audio_info = { 'bitrate': str(self.bitrate), - 'samplerate': str(self.samplerate), - 'quality': str(self.ogg_quality), - 'channels': str(self.voices),} + if self.channel.format == 'mp3': + self.channel.audio_info = { 'bitrate': str(self.bitrate), + 'samplerate': str(self.samplerate), + 'channels': str(self.voices),} + else: + self.channel.audio_info = { 'bitrate': str(self.bitrate), + 'samplerate': str(self.samplerate), + 'quality': str(self.ogg_quality), + 'channels': str(self.voices),} self.server_url = 'http://' + self.channel.host + ':' + str(self.channel.port) self.channel_url = self.server_url + self.channel.mount From 4bc780bddfca578b4d2d91897caf25b268a3007f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20LASSERRE?= Date: Wed, 6 Aug 2014 16:15:14 +0200 Subject: [PATCH 08/82] Delete CHANGELOG Remove old CHANGELOG file. --- CHANGELOG | 109 ------------------------------------------------------ 1 file changed, 109 deletions(-) delete mode 100644 CHANGELOG diff --git a/CHANGELOG b/CHANGELOG deleted file mode 100644 index 509889d..0000000 --- a/CHANGELOG +++ /dev/null @@ -1,109 +0,0 @@ -deefuzzer (0.6.3-1) unstable; urgency=low - - * setup rewritten - * closes: #1 - -deefuzzer (0.6.2-1) unstable; urgency=low - - * No new functions but bugfixes (including a serious one during install from pypi) - * Definitely moved the project to `GitHub `_ - * Update various README details - * update API doc: http://files.parisson.com/doc/deefuzzer/ - - -deefuzzer (0.6.1-1) unstable; urgency=low - -* new HTTP steamer based on pycurl to handle stream-m servers (webm streaming) - see http://code.google.com/p/stream-m/ -* live webm relaying works good, webm playlist reading NEED testing -* new parameter ('icecast or 'stream-m') - --- Guillaume Pellerin Thu, 12 Jan 2012 13:22:22 +0100 - -For older changes:: - - * 10/24/2011: - - Follow now the !DeeFuzzer news from the blog, right on this page.[[BR]] - Checkout the [http://svn.parisson.org/deefuzzer/blog?format=rss RSS feed] ! - - * 09/27/2011: - - Sorry the 0.5.8 install procedure was just wrong ! - Now fixed in [http://pypi.python.org/packages/source/D/DeeFuzzer/DeeFuzzer-0.6.tar.gz DeeFuzzer 0.6] - - * 09/26/2011: - - [http://debian.parisson.org/DeeFuzzer-0.5.8.tar.gz DeeFuzzer 0.5.8] has been released ! [[BR]] - [http://svn.parisson.org/deefuzzer/log/?action=stop_on_copy&mode=stop_on_copy&rev=285&stop_rev=277&limit=100 Changelog].[[BR]] - - * 05/30/2011: - - [http://debian.parisson.org/deefuzzer-0.5.4.tar.gz DeeFuzzer 0.5.4] has been released ! [[BR]] - bugfixes and minor changes. [[BR]] - DeeFuzzer is now is real module and [http://pypi.python.org/pypi?:action=display&name=DeeFuzzer&version=0.5.4 part of Python Package Index] [[BR]] - [http://svn.parisson.org/deefuzzer/log/?action=stop_on_copy&mode=stop_on_copy&rev=266&stop_rev=256&limit=100 Changelog].[[BR]] - - * December 18th 2010 Merry Xmas !: - - [http://debian.parisson.org/deefuzzer-0.5.0.tar.gz DeeFuzzer 0.5.0] has been released ! [[BR]] - - Fix the twitting module against twitter oauth API ! [[BR]] - You'll need your own token keys now. See INSTALL. [[BR]] - Be careful, XML tags have changed for twitter keys. [[BR]] - Please update you XML config file if you use the twitter module.[[BR]] - [http://svn.parisson.org/deefuzzer/log/?action=stop_on_copy&mode=stop_on_copy&rev=256&stop_rev=250&limit=100 Changelog].[[BR]] - - * December 6th 2010: - - [http://debian.parisson.org/deefuzzer-0.4.3.tar.gz DeeFuzzer 0.4.3] has been released ! [[BR]] - - Mainly bugfixes again : [http://svn.parisson.org/deefuzzer/log/?action=stop_on_copy&mode=stop_on_copy&rev=248&stop_rev=237&limit=100 changelog here].[[BR]] - Bugfixes for the metadata writing method, improve buffer behavior when no network is available.[[BR]] - As python-twitter is not updated yet against the new Twitter API OAuth, the !DeeFuzzer twitter posting method is disabled again in this version :( - - * September 9th 2010: - - [http://debian.parisson.org/deefuzzer-0.4.2.tar.gz DeeFuzzer 0.4.2] has been released ! [[BR]] - - Mainly bugfixes : [http://svn.parisson.org/deefuzzer/log/?action=stop_on_copy&mode=stop_on_copy&rev=236&stop_rev=210&limit=100 changelog here].[[BR]] - Improve the Recorder which has been tested in production during 150 record sessions (450 hours tot), seems OK ! - - Due to the new oauth system provided by Twitter, our old based authorization do NOT work anymore.[[BR]] - So please DISABLE the twitter mode of your DeeFuzzer until the next really soon release (python-twitter-0.8 needed and coming...) - - Stay Tuned ! - - * March 2nd 2010: - - [http://debian.parisson.org/deefuzzer-0.4.1.tar.gz DeeFuzzer 0.4.1] has been released ! [[BR]] - New features : recording, full documented example [[BR]] - Fix many bugs : [http://svn.parisson.org/deefuzzer/log/?action=stop_on_copy&mode=stop_on_copy&rev=208&stop_rev=185&limit=100 Changelog here]. [[BR]] - Stable here for 12 stations during 1 month :) - - * December 24th 2009: - - [http://debian.parisson.org/deefuzzer-0.4.tar.gz DeeFuzzer 0.4] has been released ! [[BR]] - New features : twitting, jingling, relaying, all controlled by OSC commands [[BR]] - [http://svn.parisson.org/deefuzzer/log/?action=stop_on_copy&mode=stop_on_copy&rev=184&stop_rev=138&limit=100 Changelog here]. [[BR]] - Happy X-Mas ;-) - - * October 5th 2009: - - [http://debian.parisson.org/deefuzzer-0.3.2.tar.gz DeeFuzzer 0.3.2] has been released ! [http://svn.parisson.org/deefuzzer/log/?action=stop_on_copy&mode=stop_on_copy&rev=138&stop_rev=134&limit=100 Changelog here]. - - - * April 17th 2009: - - [http://debian.parisson.org/deefuzzer-0.3.1.tar.gz DeeFuzzer 0.3.1] has been released ! [http://debian.parisson.org/deefuzzer-0.3.1.tar.gz Get it] and !DeeFuzz it ;) - - * April 16th 2009: - - !DeeFuzzer is now more stable thanks to the new threading capability of shout-python and many bugfixes.[[BR]] - It is now able to start multiple stations at one time without xruns...[[BR]] - I will release 0.3.0 very soon as a beta version... Enjoy ! ;-) - - * March 29th 2009: - - !DeeFuzzer is now fully functional for one station thread (new RSS feeding). - From 60de1f14c626925e9e35c961130602bb43f83a2e Mon Sep 17 00:00:00 2001 From: Guillaume Pellerin Date: Wed, 6 Aug 2014 19:56:42 +0200 Subject: [PATCH 09/82] begin station_dir parameter --- deefuzzer/station.py | 4 ++++ deefuzzer/tools/utils.py | 2 ++ 2 files changed, 6 insertions(+) diff --git a/deefuzzer/station.py b/deefuzzer/station.py index 53fe6e9..6219acd 100644 --- a/deefuzzer/station.py +++ b/deefuzzer/station.py @@ -71,6 +71,7 @@ class Station(Thread): relay_mode = 0 record_mode = 0 run_mode = 1 + station_dir = None def __init__(self, station, q, logger, m3u): Thread.__init__(self) @@ -79,6 +80,9 @@ def __init__(self, station, q, logger, m3u): self.logger = logger self.m3u = m3u + if 'station_dir' in self.station: + self.station_dir = self.station['station_dir'] + # Media self.media_dir = self.station['media']['dir'] self.media_format = self.station['media']['format'] diff --git a/deefuzzer/tools/utils.py b/deefuzzer/tools/utils.py index ce24e4c..df4955b 100644 --- a/deefuzzer/tools/utils.py +++ b/deefuzzer/tools/utils.py @@ -35,3 +35,5 @@ def get_file_info(media): file_ext = file_name.split('.')[-1] return file_name, file_title, file_ext +def is_absolute_path(path): + return os.sep == path[0] From c33a468db2c6c20c0cc30307818bb93df625f947 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20LASSERRE?= Date: Wed, 6 Aug 2014 19:59:03 +0200 Subject: [PATCH 10/82] Add 0.6.6 Fix "internet" to "Internet" Remove between ":", "!" Add Uppercase --- README.rst | 57 ++++++++++++++++++++++++++++++------------------------ 1 file changed, 32 insertions(+), 25 deletions(-) diff --git a/README.rst b/README.rst index 3c134e2..cea6454 100644 --- a/README.rst +++ b/README.rst @@ -3,22 +3,22 @@ Introduction ============ -DeeFuzzer is an open, light and instant software made for streaming audio and video over internet. +DeeFuzzer is an open, light and instant software made for streaming audio and video over Internet. It is dedicated to media streaming communities who wants to create web radios, web televisions, live multimedia relaying or even as a personal home radio, with rich media contents including track metadata. Here are the main features of the deefuzzer: - * MP3, OGG Vorbis file and live streaming over internet + * MP3, OGG Vorbis file and live streaming over Internet * Full metadata encapsulation and management * RSS podcast generator (current tracks and playlists) * M3U playlist generator * Recursive, random (shuffled) or pre-defined playlists - * Multi-threaded architecture : multiple station streaming with one config file + * Multi-threaded architecture: multiple station streaming with one config file * Auto Twitter posting of the current playing and new tracks * Jingling between main tracks - * OSC controller : control the main functions from a distant terminal - * Relaying : relay and stream *LIVE* sessions ! + * OSC controller: control the main functions from a distant terminal + * Relaying: relay and stream *LIVE* sessions! * Very light and optimized streaming process * Fully written in Python * Works with Icecast2, ShoutCast, Stream-m @@ -32,36 +32,43 @@ Because our aim is to get DeeFuzzer as light as possible it is NOT capable of re News ===== +0.6.6 + + * Update station name (remove ": http://url") + * Update mountpoint name (remove .mp3 or .ogg) + * Update metadata (replace " : " by " - " between Artist and Track) + * Remove "ogg_quality" on mp3 streams + 0.6.5 - * stable WebM live streaming through Stream-m server - * read yaml configuration files - * read m3u playlist files - * minor fixes + * Stable WebM live streaming through Stream-m server + * Read yaml configuration files + * Read m3u playlist files + * Minor fixes 0.6.4 is out! * Fix install bug again (add main script to install), sorry :( - * reduce streaming buffer length + * Reduce streaming buffer length -0.6.3 Fix install bug ! +0.6.3 Fix install bug! - * setup rewritten - * fix MANIFEST + * Setup rewritten + * Fix MANIFEST -0.6.2 has been released ! +0.6.2 has been released! * No new functions but bugfixes (including a serious one during install from pypi) * Definitely moved the project to `GitHub `_ * Update various README details * update API doc: http://files.parisson.com/doc/deefuzzer/ -0.6.1 is out ! +0.6.1 is out! - * new HTTP steamer based on pycurl to handle streaming over stream-m servers (WebM streaming) + * New HTTP steamer based on pycurl to handle streaming over stream-m servers (WebM streaming) see http://code.google.com/p/stream-m/ - * live webm relaying works good, webm playlist reading NEED testing - * new parameter ('icecast or 'stream-m') + * Live webm relaying works good, webm playlist reading NEED testing + * New parameter ('icecast or 'stream-m') Installation @@ -108,7 +115,7 @@ For more informations, please see on `GitHub `_ -Then any OSC remote (PureDate, Monome, TouchOSC, etc..) can a become controller ! :) +Then any OSC remote (PureDate, Monome, TouchOSC, etc..) can a become controller! :) We provide some client python scripts as some examples about how to control the parameters from a console or any application (see deefuzzer/scripts/). @@ -186,7 +193,7 @@ Please run the dedicated script to do this from the main deefuzzer app directory You will be invited to copy/paste an URL in your browser to get a pin code. Then copy/paste this code into the console and press ENTER. -The script gives you a pair of keys : one access token key and one access token secret key. +The script gives you a pair of keys: one access token key and one access token secret key. Change the block options in your deefuzzer XML config file, giving the 2 keys. For example:: @@ -208,7 +215,7 @@ http://files.parisson.com/doc/deefuzzer/ Development ============ -Everybody is welcome to participate to the DeeFuzzer project ! +Everybody is welcome to participate to the DeeFuzzer project! We use GitHub to collaborate: https://github.com/yomguy/DeeFuzzer Join us! @@ -231,12 +238,12 @@ as described in the file LICENSE.txt in the source directory or online https://g Aknowledgements =============== -This work is inspired by the great - C coded - Oddsock's streaming program : Ezstream. +This work is inspired by the great - C coded - Oddsock's streaming program: Ezstream. Since I needed to patch it in order to modify the playlist (randomize for example) and make external batch tools to create multiple channels, I decided to rewrite it from scratch in python. -Some parts of this work are also taken from another Parisson's project : Telemeta +Some parts of this work are also taken from another Parisson's project: Telemeta (see http://telemeta.org). Contact / Infos @@ -244,6 +251,6 @@ Contact / Infos Twitter: @yomguy @parisson_studio -GitHub : https://github.com/yomguy/DeeFuzzer +GitHub: https://github.com/yomguy/DeeFuzzer Expertise, Business, Sponsoring: http://parisson.com From 3706189807eda735ea53f840ceb748f5a1c78a59 Mon Sep 17 00:00:00 2001 From: Guillaume Pellerin Date: Thu, 7 Aug 2014 11:59:38 +0200 Subject: [PATCH 11/82] cleanup Manifest (fix #19) --- MANIFEST.in | 1 - 1 file changed, 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index 6b13a1a..2c6e3a9 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,3 @@ -include CHANGELOG include *.txt include *.rst exclude *.cfg From a28e0097d19903146aa77b6c6ab1344f15c921ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20LASSERRE?= Date: Wed, 13 Aug 2014 21:52:27 +0200 Subject: [PATCH 12/82] Remove unnecessary variables --- deefuzzer/core.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/deefuzzer/core.py b/deefuzzer/core.py index a909f93..ef0f1fc 100644 --- a/deefuzzer/core.py +++ b/deefuzzer/core.py @@ -107,14 +107,11 @@ def set_m3u_playlist(self): m3u = open(self.m3u, 'w') m3u.write('#EXTM3U\n') for s in self.stations: - info = '#EXTINF:%s,%s - %s\n' % ('-1',s.short_name, s.channel.name) - url = 'http://' + s.channel.host + ':' + str(s.channel.port) + s.channel.mount + '\n' - m3u.write(info) - m3u.write(url) + m3u.write('#EXTINF:%s,%s - %s\n' % ('-1',s.short_name, s.channel.name)) + m3u.write('http://' + s.channel.host + ':' + str(s.channel.port) + s.channel.mount + '\n') m3u.close() self.logger.write_info('Writing M3U file to : ' + self.m3u) - def run(self): q = Queue.Queue(1) @@ -149,4 +146,3 @@ def run(self): while True: q.put(i,1) i+=1 - From 96c3c96633302c6117b84c63714cfadf3cf43f3f Mon Sep 17 00:00:00 2001 From: achbed Date: Wed, 19 Nov 2014 03:18:33 -0600 Subject: [PATCH 13/82] Add option to append stream type to the mount path for icecast streams. Default is enabled (matches current behavior) Signed-off-by: achbed --- deefuzzer/station.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/deefuzzer/station.py b/deefuzzer/station.py index 6cb020c..1b04c8b 100644 --- a/deefuzzer/station.py +++ b/deefuzzer/station.py @@ -72,6 +72,7 @@ class Station(Thread): record_mode = 0 run_mode = 1 station_dir = None + appendtype = 1 def __init__(self, station, q, logger, m3u): Thread.__init__(self) @@ -105,6 +106,9 @@ def __init__(self, station, q, logger, m3u): self.short_name = self.mountpoint + if 'appendtype' in self.station['server'].keys(): + self.appendtype = self.station['server']['appendtype'] + if 'type' in self.station['server']: self.type = self.station['server']['type'] # 'icecast' | 'stream-m' else: @@ -115,7 +119,9 @@ def __init__(self, station, q, logger, m3u): self.channel.mount = '/publish/' + self.mountpoint elif 'icecast' in self.type: self.channel = shout.Shout() - self.channel.mount = '/' + self.mountpoint + '.' + self.media_format + self.channel.mount = '/' + self.mountpoint + if self.appendtype: + self.channel.mount = self.channel.mount + '.' + self.media_format else: sys.exit('Not a compatible server type. Choose "stream-m" or "icecast".') From 511156bbe1f1cc1eca9da9009cdd009f36fc2c81 Mon Sep 17 00:00:00 2001 From: achbed Date: Wed, 19 Nov 2014 11:00:24 -0600 Subject: [PATCH 14/82] Added examples for appendtype Signed-off-by: achbed --- example/deefuzzer.json | 3 ++- example/deefuzzer.xml | 1 + example/deefuzzer.yaml | 2 +- example/deefuzzer_doc.xml | 3 +++ 4 files changed, 7 insertions(+), 2 deletions(-) diff --git a/example/deefuzzer.json b/example/deefuzzer.json index fd4acfe..caca5f6 100644 --- a/example/deefuzzer.json +++ b/example/deefuzzer.json @@ -49,7 +49,8 @@ "port": 8000, "public": 0, "sourcepassword": "icecast_source_password", - "type": "icecast" + "type": "icecast", + "appendtype": 1 }, "twitter": { "key": "you access token key", diff --git a/example/deefuzzer.xml b/example/deefuzzer.xml index c3f48bf..9db8985 100644 --- a/example/deefuzzer.xml +++ b/example/deefuzzer.xml @@ -49,6 +49,7 @@ 0 icecast_source_password icecast + 1 your access token key diff --git a/example/deefuzzer.yaml b/example/deefuzzer.yaml index d663020..8e72068 100644 --- a/example/deefuzzer.yaml +++ b/example/deefuzzer.yaml @@ -40,7 +40,7 @@ deefuzzer: mountpoint: monitor, port: 8000, public: 0, sourcepassword: icecast_source_password, - type: icecast} + type: icecast, appendtype: 1} twitter: {key: 'your access token key', mode: 0, secret: 'your acess token secret key', diff --git a/example/deefuzzer_doc.xml b/example/deefuzzer_doc.xml index aa915be..cc7190c 100644 --- a/example/deefuzzer_doc.xml +++ b/example/deefuzzer_doc.xml @@ -94,6 +94,9 @@ icecast_source_password icecast + + 1 From b4072eb514d71eed345c6d96b19d2c87a47b1e09 Mon Sep 17 00:00:00 2001 From: achbed Date: Wed, 19 Nov 2014 11:03:04 -0600 Subject: [PATCH 15/82] tab to space (for indent issues) Signed-off-by: achbed --- deefuzzer/station.py | 6 +++--- example/deefuzzer.json | 2 +- example/deefuzzer.xml | 2 +- example/deefuzzer_doc.xml | 6 +++--- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/deefuzzer/station.py b/deefuzzer/station.py index 1b04c8b..abc10a0 100644 --- a/deefuzzer/station.py +++ b/deefuzzer/station.py @@ -72,7 +72,7 @@ class Station(Thread): record_mode = 0 run_mode = 1 station_dir = None - appendtype = 1 + appendtype = 1 def __init__(self, station, q, logger, m3u): Thread.__init__(self) @@ -120,8 +120,8 @@ def __init__(self, station, q, logger, m3u): elif 'icecast' in self.type: self.channel = shout.Shout() self.channel.mount = '/' + self.mountpoint - if self.appendtype: - self.channel.mount = self.channel.mount + '.' + self.media_format + if self.appendtype: + self.channel.mount = self.channel.mount + '.' + self.media_format else: sys.exit('Not a compatible server type. Choose "stream-m" or "icecast".') diff --git a/example/deefuzzer.json b/example/deefuzzer.json index caca5f6..4695b5e 100644 --- a/example/deefuzzer.json +++ b/example/deefuzzer.json @@ -50,7 +50,7 @@ "public": 0, "sourcepassword": "icecast_source_password", "type": "icecast", - "appendtype": 1 + "appendtype": 1 }, "twitter": { "key": "you access token key", diff --git a/example/deefuzzer.xml b/example/deefuzzer.xml index 9db8985..e482e72 100644 --- a/example/deefuzzer.xml +++ b/example/deefuzzer.xml @@ -49,7 +49,7 @@ 0 icecast_source_password icecast - 1 + 1 your access token key diff --git a/example/deefuzzer_doc.xml b/example/deefuzzer_doc.xml index cc7190c..4bc8253 100644 --- a/example/deefuzzer_doc.xml +++ b/example/deefuzzer_doc.xml @@ -94,9 +94,9 @@ icecast_source_password icecast - - 1 + + 1 From 6bcffb63f385d47e09c82e8eeecdc1bc3f761572 Mon Sep 17 00:00:00 2001 From: achbed Date: Wed, 19 Nov 2014 14:48:19 -0600 Subject: [PATCH 16/82] Added ability to export json data as well as RSS feeds Added ability to selectively enable/disable json or RSS output Renamed tag to RSS reads from tag if no tag is present (for backwards compatibility) Signed-off-by: achbed --- deefuzzer/station.py | 56 ++++++++++++++++++++++++++++++++------------ 1 file changed, 41 insertions(+), 15 deletions(-) diff --git a/deefuzzer/station.py b/deefuzzer/station.py index 6cb020c..b9b1810 100644 --- a/deefuzzer/station.py +++ b/deefuzzer/station.py @@ -72,6 +72,8 @@ class Station(Thread): record_mode = 0 run_mode = 1 station_dir = None + feed_json = 0 + feed_rss = 1 def __init__(self, station, q, logger, m3u): Thread.__init__(self) @@ -145,20 +147,30 @@ def __init__(self, station, q, logger, m3u): self.channel_url = self.server_url + self.channel.mount # RSS - if 'rss' in self.station: - if 'mode' in self.station['rss']: - self.rss_mode = int(self.station['rss']['mode']) + rss_options = {} + if 'feed' in self.station: + rss_options['rss'] = self.station['feed'] + elif 'rss' in self.station: + rss_options['rss'] = self.station['rss'] + + self.rss_media_url = self.channel.url + '/media/' + + if 'rss' in rss_options: + if 'mode' in rss_options['rss']: + self.rss_mode = int(rss_options['rss']['mode']) else: self.rss_mode = 0 - self.rss_dir = self.station['rss']['dir'] - self.rss_enclosure = self.station['rss']['enclosure'] - - if 'media_url' in self.station['rss']: - self.rss_media_url = self.station['rss']['media_url'] - if not self.rss_media_url: - self.rss_media_url = self.channel.url + '/media/' - else: - self.rss_media_url = self.channel.url + '/media/' + self.rss_dir = rss_options['rss']['dir'] + self.rss_enclosure = int(rss_options['rss']['enclosure']) + if 'json' in rss_options['rss']: + self.feed_json = int(rss_options['rss']['json']) + if 'rss' in rss_options['rss']: + self.feed_rss = int(rss_options['rss']['rss']) + + if 'media_url' in rss_options['rss']: + self.rss_media_url = rss_options['rss']['media_url'] + if not self.rss_media_url: + self.rss_media_url = self.channel.url + '/media/' self.base_name = self.rss_dir + os.sep + self.short_name + '_' + self.channel.format self.rss_current_file = self.base_name + '_current.xml' @@ -472,8 +484,10 @@ def update_rss(self, media_list, rss_file, sub_title): _date_now = datetime.datetime.now() date_now = str(_date_now) media_absolute_playtime = _date_now + json_data = [] for media in media_list: + json_item = {} media_stats = os.stat(media.media) media_date = time.localtime(media_stats[8]) media_date = time.strftime("%a, %d %b %Y %H:%M:%S +0200", media_date) @@ -487,6 +501,7 @@ def update_rss(self, media_list, rss_file, sub_title): if media.metadata[key] != '': media_description += media_description_item % (key.capitalize(), media.metadata[key]) + json_item[key] = media.metadata[key] media_description += '' title = media.metadata['title'] @@ -522,15 +537,26 @@ def update_rss(self, media_list, rss_file, sub_title): guid = Guid(media_link), pubDate = media_date,) ) + json_data.append(json_item) rss = RSS2(title = channel_subtitle, link = self.channel.url, description = self.channel.description.decode('utf-8'), lastBuildDate = date_now, items = rss_item_list,) - f = open(rss_file, 'w') - rss.write_xml(f, 'utf-8') - f.close() + + if self.feed_rss: + f = open(rss_file, 'w') + rss.write_xml(f, 'utf-8') + f.close() + + if self.feed_json: + path, fn = os.path.split(rss_file) + base, ext = os.path.splitext(fn) + json_file = os.path.join(self.rss_dir, base + '.json') + f = open(json_file, 'w') + f.write(json.dumps(json_data, separators=(',',':'))) + f.close() def update_twitter(self, message): try: From 8d8920f21cf03c271cfff3988e8b77d696f0053c Mon Sep 17 00:00:00 2001 From: achbed Date: Wed, 19 Nov 2014 15:46:32 -0600 Subject: [PATCH 17/82] Missing json import Moved handling of rss_mode to within the update_rss function Renamed rss_* variables to feeds_* Renamed feed_* variables to feeds_* Renamed update_rss to update_feeds Fixed feeds_mode to have a default of 1 (enabled) Fixed issue where playlist feeds were not created at start time Added ability to disable playlist feed output Added tag to feeds options (1=output,0=suppress,default=1) Signed-off-by: achbed --- deefuzzer/station.py | 93 ++++++++++++++++++++++---------------------- 1 file changed, 47 insertions(+), 46 deletions(-) diff --git a/deefuzzer/station.py b/deefuzzer/station.py index b9b1810..82f418b 100644 --- a/deefuzzer/station.py +++ b/deefuzzer/station.py @@ -45,6 +45,7 @@ import shout import urllib import mimetypes +import json from threading import Thread from player import * @@ -72,8 +73,10 @@ class Station(Thread): record_mode = 0 run_mode = 1 station_dir = None - feed_json = 0 - feed_rss = 1 + feeds_json = 0 + feeds_rss = 1 + feeds_mode = 1 + feeds_playlist = 1 def __init__(self, station, q, logger, m3u): Thread.__init__(self) @@ -147,34 +150,29 @@ def __init__(self, station, q, logger, m3u): self.channel_url = self.server_url + self.channel.mount # RSS - rss_options = {} - if 'feed' in self.station: - rss_options['rss'] = self.station['feed'] - elif 'rss' in self.station: - rss_options['rss'] = self.station['rss'] - - self.rss_media_url = self.channel.url + '/media/' - - if 'rss' in rss_options: - if 'mode' in rss_options['rss']: - self.rss_mode = int(rss_options['rss']['mode']) - else: - self.rss_mode = 0 - self.rss_dir = rss_options['rss']['dir'] - self.rss_enclosure = int(rss_options['rss']['enclosure']) - if 'json' in rss_options['rss']: - self.feed_json = int(rss_options['rss']['json']) - if 'rss' in rss_options['rss']: - self.feed_rss = int(rss_options['rss']['rss']) - - if 'media_url' in rss_options['rss']: - self.rss_media_url = rss_options['rss']['media_url'] - if not self.rss_media_url: - self.rss_media_url = self.channel.url + '/media/' - - self.base_name = self.rss_dir + os.sep + self.short_name + '_' + self.channel.format - self.rss_current_file = self.base_name + '_current.xml' - self.rss_playlist_file = self.base_name + '_playlist.xml' + if 'feeds' in self.station: + self.station['rss'] = self.station['feeds'] + + if 'rss' in self.station: + if 'mode' in self.station['rss']: + self.feeds_mode = int(self.station['rss']['mode']) + self.feeds_dir = self.station['rss']['dir'] + self.feeds_enclosure = int(self.station['rss']['enclosure']) + if 'json' in self.station['rss']: + self.feeds_json = int(self.station['rss']['json']) + if 'rss' in self.station['rss']: + self.feeds_rss = int(self.station['rss']['rss']) + if 'playlist' in self.station['rss']: + self.feeds_playlist = int(self.station['rss']['playlist']) + + self.feeds_media_url = self.channel.url + '/media/' + if 'media_url' in self.station['rss']: + if self.station['rss']['media_url']: + self.feeds_media_url = self.station['rss']['media_url'] + + self.base_name = self.feeds_dir + os.sep + self.short_name + '_' + self.channel.format + self.feeds_current_file = self.base_name + '_current.xml' + self.feeds_playlist_file = self.base_name + '_playlist.xml' # Logging self.logger.write_info('Opening ' + self.short_name + ' - ' + self.channel.name + \ @@ -182,7 +180,7 @@ def __init__(self, station, q, logger, m3u): self.metadata_relative_dir = 'metadata' self.metadata_url = self.channel.url + '/rss/' + self.metadata_relative_dir - self.metadata_dir = self.rss_dir + os.sep + self.metadata_relative_dir + self.metadata_dir = self.feeds_dir + os.sep + self.metadata_relative_dir if not os.path.exists(self.metadata_dir): os.makedirs(self.metadata_dir) @@ -286,7 +284,7 @@ def twitter_callback(self, path, value): message = "Station " + self.channel_url + \ " : received OSC message '%s' with arguments '%d'" % (path, value) self.m3u_url = self.channel.url + '/m3u/' + self.m3u.split(os.sep)[-1] - self.rss_url = self.channel.url + '/rss/' + self.rss_playlist_file.split(os.sep)[-1] + self.feeds_url = self.channel.url + '/rss/' + self.feeds_playlist_file.split(os.sep)[-1] self.logger.write_info(message) def jingles_callback(self, path, value): @@ -390,7 +388,7 @@ def tweet(self): artist_tags = ' #'.join(list(set(artist_names)-set(['&', '-']))) message = '#NEWTRACK ! %s #%s on #%s RSS: ' % \ (song.replace('_', ' '), artist_tags, self.short_name) - message = message[:113] + self.rss_url + message = message[:113] + self.feeds_url self.update_twitter(message) def get_next_media(self): @@ -412,6 +410,8 @@ def get_next_media(self): self.logger.write_info('Station ' + self.channel_url + \ ' : generating new playlist (' + str(self.lp) + ' tracks)') + if self.feeds_playlist: + self.update_feeds(self.media_to_objs(self.playlist), self.feeds_playlist_file, '(playlist)') elif lp_new != self.lp: self.id = 0 @@ -439,9 +439,8 @@ def get_next_media(self): self.logger.write_info('Station ' + self.channel_url + \ ' : generating new playlist (' + str(self.lp) + ' tracks)') - if self.rss_mode: - self.update_rss(self.media_to_objs(self.playlist), - self.rss_playlist_file, '(playlist)') + if self.feeds_playlist: + self.update_feeds(self.media_to_objs(self.playlist), self.feeds_playlist_file, '(playlist)') if self.jingles_mode and not (self.counter % 2) and self.jingles_length: media = self.jingles_list[self.jingle_id] @@ -476,10 +475,13 @@ def media_to_objs(self, media_list): continue return media_objs - def update_rss(self, media_list, rss_file, sub_title): + def update_feeds(self, media_list, rss_file, sub_title): + if not self.feeds_mode: + return + rss_item_list = [] - if not os.path.exists(self.rss_dir): - os.makedirs(self.rss_dir) + if not os.path.exists(self.feeds_dir): + os.makedirs(self.feeds_dir) channel_subtitle = self.channel.name + ' ' + sub_title _date_now = datetime.datetime.now() date_now = str(_date_now) @@ -513,8 +515,8 @@ def update_rss(self, media_list, rss_file, sub_title): media_absolute_playtime += media.length - if self.rss_enclosure == '1': - media_link = self.rss_media_url + media.file_name + if self.feeds_enclosure == '1': + media_link = self.feeds_media_url + media.file_name media_link = media_link.decode('utf-8') rss_item_list.append(RSSItem( title = song, @@ -545,15 +547,15 @@ def update_rss(self, media_list, rss_file, sub_title): lastBuildDate = date_now, items = rss_item_list,) - if self.feed_rss: + if self.feeds_rss: f = open(rss_file, 'w') rss.write_xml(f, 'utf-8') f.close() - if self.feed_json: + if self.feeds_json: path, fn = os.path.split(rss_file) base, ext = os.path.splitext(fn) - json_file = os.path.join(self.rss_dir, base + '.json') + json_file = os.path.join(self.feeds_dir, base + '.json') f = open(json_file, 'w') f.write(json.dumps(json_data, separators=(',',':'))) f.close() @@ -603,8 +605,7 @@ def set_read_mode(self): self.song = song.encode('utf-8') self.artist = self.artist.encode('utf-8') self.metadata_file = self.metadata_dir + os.sep + self.current_media_obj[0].file_name + '.xml' - if self.rss_mode: - self.update_rss(self.current_media_obj, self.rss_current_file, '(currently playing)') + self.update_feeds(self.current_media_obj, self.feeds_current_file, '(currently playing)') self.logger.write_info('DeeFuzzing on %s : id = %s, name = %s' \ % (self.short_name, self.id, self.current_media_obj[0].file_name)) self.player.set_media(self.media) From 12ff319dff76084425e733c4ad45bdab26e60040 Mon Sep 17 00:00:00 2001 From: achbed Date: Wed, 19 Nov 2014 15:57:41 -0600 Subject: [PATCH 18/82] Updated example config files with new/renamed options Signed-off-by: achbed --- example/deefuzzer.json | 6 +++++- example/deefuzzer.xml | 8 ++++++-- example/deefuzzer.yaml | 3 ++- example/deefuzzer_doc.xml | 14 +++++++++++--- 4 files changed, 24 insertions(+), 7 deletions(-) diff --git a/example/deefuzzer.json b/example/deefuzzer.json index fd4acfe..a535be2 100644 --- a/example/deefuzzer.json +++ b/example/deefuzzer.json @@ -38,7 +38,11 @@ "mode": 0, "url": "http://127.0.0.1:8000/telecaster_live.mp3" }, - "rss": { + "feeds": { + "mode": 1, + "rss": 1, + "json": 0, + "playlist": 0, "dir": "/path/to/rss", "enclosure": 1, "media_url": "http://localhost/media/" diff --git a/example/deefuzzer.xml b/example/deefuzzer.xml index c3f48bf..cdc04ec 100644 --- a/example/deefuzzer.xml +++ b/example/deefuzzer.xml @@ -37,11 +37,15 @@ 0 http://127.0.0.1:8000/telecaster_live.mp3 - + + 1 + 1 + 0 + 1 /path/to/rss/ 1 http://localhost/media/ - + 127.0.0.1 monitor diff --git a/example/deefuzzer.yaml b/example/deefuzzer.yaml index d663020..8987c37 100644 --- a/example/deefuzzer.yaml +++ b/example/deefuzzer.yaml @@ -32,7 +32,8 @@ deefuzzer: mode: 0, url: 'http://127.0.0.1:8000/telecaster_live.mp3'} - rss: {dir: /var/www/rss, + feeds: {mode: 1, rss: 1, json: 0, playlist: 1, + dir: /var/www/rss, enclosure: 0, media_url: 'http://localhost/media/'} diff --git a/example/deefuzzer_doc.xml b/example/deefuzzer_doc.xml index aa915be..92ce38d 100644 --- a/example/deefuzzer_doc.xml +++ b/example/deefuzzer_doc.xml @@ -69,15 +69,23 @@ http://127.0.0.1:8000/telecaster_live.mp3 - + 1 + + 1 + + 0 + + 1 + /path/to/rss/ - 1 - http://localhost/media/ From e57de40442a114a6bc0053fa38109b80a85d373b Mon Sep 17 00:00:00 2001 From: achbed Date: Wed, 19 Nov 2014 15:59:38 -0600 Subject: [PATCH 19/82] Fixed documented example with renamed feeds tag Signed-off-by: achbed --- example/deefuzzer_doc.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/example/deefuzzer_doc.xml b/example/deefuzzer_doc.xml index 92ce38d..9914317 100644 --- a/example/deefuzzer_doc.xml +++ b/example/deefuzzer_doc.xml @@ -68,7 +68,7 @@ http://127.0.0.1:8000/telecaster_live.mp3 - + 1 @@ -88,7 +88,7 @@ http://localhost/media/ - + 127.0.0.1 From 4d49e0a1cf971904f1dc3e388e7b099d61cfc86b Mon Sep 17 00:00:00 2001 From: achbed Date: Wed, 19 Nov 2014 22:52:32 -0600 Subject: [PATCH 20/82] Added stationdefaults parameter (provides a mechanism to give default values for all stations) New merge_defaults function in utils.py - combines two dict objects recursively Signed-off-by: achbed --- deefuzzer/core.py | 5 +++++ deefuzzer/tools/utils.py | 17 +++++++++++++++++ example/deefuzzer.json | 11 +++++++++++ example/deefuzzer.xml | 11 +++++++++++ example/deefuzzer.yaml | 8 ++++++++ example/deefuzzer_doc.xml | 22 ++++++++++++++++++++++ 6 files changed, 74 insertions(+) diff --git a/deefuzzer/core.py b/deefuzzer/core.py index ef0f1fc..49e636f 100644 --- a/deefuzzer/core.py +++ b/deefuzzer/core.py @@ -120,6 +120,11 @@ def run(self): station = self.conf['deefuzzer']['station'] else: station = self.conf['deefuzzer']['station'][i] + + # Apply station defaults if they exist + if 'stationdefaults' in self.conf['deefuzzer']: + if isinstance(self.conf['deefuzzer']['stationdefaults'], dict): + station = merge_defaults(station, self.conf['deefuzzer']['stationdefaults']) self.stations.append(Station(station, q, self.logger, self.m3u)) if self.m3u: diff --git a/deefuzzer/tools/utils.py b/deefuzzer/tools/utils.py index df4955b..3d2143b 100644 --- a/deefuzzer/tools/utils.py +++ b/deefuzzer/tools/utils.py @@ -13,6 +13,8 @@ import os import re import string +from itertools import chain +from deefuzzer.tools import * def clean_word(word) : """ Return the word without excessive blank spaces, underscores and @@ -37,3 +39,18 @@ def get_file_info(media): def is_absolute_path(path): return os.sep == path[0] + +def merge_defaults(setting, default): + combined = {} + for key in set(chain(setting, default)): + if key in setting: + if key in default: + if isinstance(setting[key], dict) and isinstance(default[key], dict): + combined[key] = merge_defaults(setting[key], default[key]) + else: + combined[key] = setting[key] + else: + combined[key] = setting[key] + else: + combined[key] = default[key] + return combined diff --git a/example/deefuzzer.json b/example/deefuzzer.json index cc4078e..2234ded 100644 --- a/example/deefuzzer.json +++ b/example/deefuzzer.json @@ -2,6 +2,17 @@ "deefuzzer": { "log": "/path/to/station.log", "m3u": "/path/to/station.m3u", + "station": { + "control": { + "mode": 0, + "port": 16001 + }, + "jingles": { + "dir": "/path/to/jingles", + "mode": 0, + "shuffle": 1 + } + }, "station": { "control": { "mode": 0, diff --git a/example/deefuzzer.xml b/example/deefuzzer.xml index 593d2ed..9cadbb1 100644 --- a/example/deefuzzer.xml +++ b/example/deefuzzer.xml @@ -1,6 +1,17 @@ /path/to/station.log /path/to/station.m3u + + + 0 + 16001 + + + /path/to/jingles + 0 + 1 + + 0 diff --git a/example/deefuzzer.yaml b/example/deefuzzer.yaml index 8e33454..366cb7f 100644 --- a/example/deefuzzer.yaml +++ b/example/deefuzzer.yaml @@ -2,6 +2,14 @@ deefuzzer: log: /path/to/station.log m3u: /path/to/station.m3u + stationdefaults: + control: {mode: 0, + port: 16001} + + jingles: {dir: /path/to/jingles, + mode: 0, + shuffle: 1} + station: control: {mode: 0, port: 16001} diff --git a/example/deefuzzer_doc.xml b/example/deefuzzer_doc.xml index be45286..1beb53f 100644 --- a/example/deefuzzer_doc.xml +++ b/example/deefuzzer_doc.xml @@ -5,6 +5,28 @@ The file is preferably accessible behind an url, for example, http://mydomain.com/m3u/mystation.m3u --> /path/to/station.m3u + + + + + 0 + + 16001 + + + + /path/to/jingles + + 0 + + 1 + + /path/to/station.m3u + + + + + 0 + + 16001 + + + + /path/to/jingles + + 0 + + 1 + + - - + + + + /path/to/media + + + [name] + [name] + [name] + + + [path] + + + + + /path/to/configs + /path/to/configs2 + diff --git a/example/stationconfig_doc.xml b/example/stationconfig_doc.xml new file mode 100644 index 0000000..740b3c0 --- /dev/null +++ b/example/stationconfig_doc.xml @@ -0,0 +1,36 @@ + + + + My personal best funky playlist ever! + My best funky station + My_station + http://parisson.com + Various Funk Groove + + + /path/to/jingles + 0 + 1 + + + 96 + /path/to/mp3/ + mp3 + /path/to/m3u_file + 4 + 48000 + 0 + 2 + + + 127.0.0.1 + monitor + 8000 + 0 + icecast_source_password + icecast + 1 + + From 8fd9200800c7a6492ced937dfc6951510be19b42 Mon Sep 17 00:00:00 2001 From: achbed Date: Thu, 20 Nov 2014 00:00:14 -0600 Subject: [PATCH 22/82] Typo Signed-off-by: achbed --- deefuzzer/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deefuzzer/core.py b/deefuzzer/core.py index 84c9194..eadeabf 100644 --- a/deefuzzer/core.py +++ b/deefuzzer/core.py @@ -131,7 +131,7 @@ def create_station(self, folder, options): s[i] = replace_all(options[i], d) if not 'media' in s.keys(): s['media'] = {} - s['media']['dir'] = folder: + s['media']['dir'] = folder self.conf['deefuzzer']['station'].append(s) From 5a46fae590d83add9f8642f0f375baa475ce62ba Mon Sep 17 00:00:00 2001 From: achbed Date: Thu, 20 Nov 2014 22:36:52 -0600 Subject: [PATCH 23/82] Added documentation Revised init handling of preferences for consistency Fixed issue where some boolean flags (ie, modes) were not being handled properly Signed-off-by: achbed --- deefuzzer/core.py | 141 ++++++++++++------- deefuzzer/station.py | 326 ++++++++++++++++++++++++++++--------------- 2 files changed, 304 insertions(+), 163 deletions(-) diff --git a/deefuzzer/core.py b/deefuzzer/core.py index eadeabf..9f401c5 100644 --- a/deefuzzer/core.py +++ b/deefuzzer/core.py @@ -51,51 +51,52 @@ class DeeFuzzer(Thread): """a DeeFuzzer diffuser""" + logger = None m3u = None rss = None + station = [] + stations = [] def __init__(self, conf_file): Thread.__init__(self) self.conf_file = conf_file self.conf = get_conf_dict(self.conf_file) + + # Get the log setting first (if possible) + log_file = str(self.conf.pop('log', '')) + log_dir = os.sep.join(log_file.split(os.sep)[:-1]) + if not os.path.exists(log_dir) and log_dir: + os.makedirs(m3u_dir) + self.logger = Logger(log_file) + for key in self.conf['deefuzzer'].keys(): - if key == 'log': - log_file = self.conf['deefuzzer']['log'] - log_dir = os.sep.join(log_file.split(os.sep)[:-1]) - if not os.path.exists(log_dir) and log_dir: - os.makedirs(m3u_dir) - self.logger = Logger(log_file) if key == 'm3u': - self.m3u = self.conf['deefuzzer']['m3u'] + self.m3u = str(self.conf['deefuzzer'][key]) + + elif key == 'station': + # Load station definitions from the main config file + if not isinstance(self.conf['deefuzzer'][key], list): + self.add_station(self.conf['deefuzzer'][key]) + else: + for s in self.conf['deefuzzer'][key]: + self.add_station(s) + + elif key == 'stationconfig': + # Load additional station definitions from the requested folder + self.load_stations_fromconfig(self.conf['deefuzzer'][key]) + + elif key == 'stationfolder': + # Create stations automagically from a folder structure + if isinstance(self.conf['deefuzzer'][key], dict): + self.create_stations_fromfolder(self.conf['deefuzzer'][key]) else: setattr(self, key, self.conf['deefuzzer'][key]) - # Fix wrong type data from xmltodict when one station (*) - if 'station' not in self.conf['deefuzzer']: - self.conf['deefuzzer']['station'] = [] - else: - if not isinstance(self.conf['deefuzzer']['station'], list): - s = self.conf['deefuzzer']['station'] - self.conf['deefuzzer']['station'] = [] - self.conf['deefuzzer']['station'].append(s) - - # Load additional station definitions from the requested folder - if 'stationconfig' in self.conf['deefuzzer']: - self.load_stations(self.conf['deefuzzer']['stationconfig']) - - # Create stations automagically from a folder structure - if 'stationfolder' in self.conf['deefuzzer'].keys() and isinstance(self.conf['deefuzzer']['stationfolder'], dict): - self.create_stations_fromfolder(self.conf['deefuzzer']['stationfolder']) - - self.nb_stations = len(self.conf['deefuzzer']['station']) - # Set the deefuzzer logger self.logger.write_info('Starting DeeFuzzer') self.logger.write_info('Using libshout version %s' % shout.version()) + self.logger.write_info('Number of stations : ' + str(len(self.station))) - # Init all Stations - self.stations = [] - self.logger.write_info('Number of stations : ' + str(self.nb_stations)) def set_m3u_playlist(self): m3u_dir = os.sep.join(self.m3u.split(os.sep)[:-1]) @@ -110,10 +111,19 @@ def set_m3u_playlist(self): self.logger.write_info('Writing M3U file to : ' + self.m3u) def create_stations_fromfolder(self, options): + """Scan a folder for subfolders containing media, and make stations from them all.""" + if not 'folder' in options.keys(): + # We have no folder specified. Bail. return + folder = str(options['folder']) - self.logger.write_info('Scanning folder ' + folder + ' for stations...') + if not os.path.isdir(folder): + # The specified path is not a folder. Bail. + return + + self.logger.write_info('Scanning folder ' + folder + ' for stations') + files = os.listdir(folder) for file in files: filepath = os.path.join(folder, file) @@ -122,6 +132,8 @@ def create_stations_fromfolder(self, options): self.create_station(filepath, options) def create_station(self, folder, options): + """Create a station definition for a folder given the specified options.""" + self.logger.write_info('Creating station for folder ' + folder) s = {} path, name = os.path.split(folder) @@ -132,43 +144,70 @@ def create_station(self, folder, options): if not 'media' in s.keys(): s['media'] = {} s['media']['dir'] = folder - self.conf['deefuzzer']['station'].append(s) - - - def load_stations(self, folder): - if isinstance(folder, dict): + self.add_station(s) + + def load_stations_fromconfig(self, folder): + """Load one or more configuration files looking for stations.""" + + if isinstance(folder, dict) or isinstance(folder, list): + # We were given a list or dictionary. Loop though it and load em all for f in folder: - self.load_stations(f) + self.load_station_configs(f) return + if os.path.isfile(folder): + # We have a file specified. Load just that file. + self.load_station_config(folder) + return + if not os.path.isdir(folder): + # Whatever we have, it's not either a file or folder. Bail. return - + self.logger.write_info('Loading station config files in ' + folder) files = os.listdir(folder) for file in files: filepath = os.path.join(folder, file) if os.path.isfile(filepath): self.load_station_config(filepath) - + def load_station_config(self, file): + """Load station configuration(s) from a config file.""" + self.logger.write_info('Loading station config file ' + file) stationdef = get_conf_dict(file) if isinstance(stationdef, dict): - if 'station' in stationdef.keys() and isinstance(stationdef['station'], dict): - self.conf['deefuzzer']['station'].append(stationdef['station']) + if 'station' in stationdef.keys(): + if isinstance(stationdef['station'], dict): + self.add_station(stationdef['station']) + elif isinstance(stationdef['station'], list): + for s in stationdef['station']: + self.add_station(s) + + def add_station(self, this_station): + """Adds a station configuration to the list of stations.""" + try: + # We should probably test to see if we're putting the same station in multiple times + # Same in this case probably means the same media folder, server, and mountpoint + self.station.append(this_station) + except Exception: + return def run(self): q = Queue.Queue(1) - for i in range(0,self.nb_stations): - station = self.conf['deefuzzer']['station'][i] + ns = len(self.station) + for i in range(0, ns): + try: + station = self.station[i] - # Apply station defaults if they exist - if 'stationdefaults' in self.conf['deefuzzer']: - if isinstance(self.conf['deefuzzer']['stationdefaults'], dict): - station = merge_defaults(station, self.conf['deefuzzer']['stationdefaults']) - self.stations.append(Station(station, q, self.logger, self.m3u)) + # Apply station defaults if they exist + if 'stationdefaults' in self.conf['deefuzzer']: + if isinstance(self.conf['deefuzzer']['stationdefaults'], dict): + station = merge_defaults(station, self.conf['deefuzzer']['stationdefaults']) + self.stations.append(Station(station, q, self.logger, self.m3u)) + except Exception: + continue if self.m3u: self.set_m3u_playlist() @@ -176,9 +215,13 @@ def run(self): p = Producer(q) p.start() + ns = len(self.stations) # Start the Stations - for i in range(0,self.nb_stations): - self.stations[i].start() + for i in range(0, ns): + try: + self.stations[i].start() + except Exception: + continue class Producer(Thread): diff --git a/deefuzzer/station.py b/deefuzzer/station.py index f1fec09..20f0b75 100644 --- a/deefuzzer/station.py +++ b/deefuzzer/station.py @@ -67,10 +67,16 @@ class Station(Thread): lp = 1 player_mode = 0 osc_control_mode = 0 + osc_control_port = 16001 twitter_mode = 0 jingles_mode = 0 + jingles_shuffle = 1 + jingles_dir = '' relay_mode = 0 + relay_url = '' + relay_author = '' record_mode = 0 + record_dir = '' run_mode = 1 station_dir = None appendtype = 1 @@ -78,6 +84,11 @@ class Station(Thread): feeds_rss = 1 feeds_mode = 1 feeds_playlist = 1 + media_dir = '' + m3u_playlist_file = [] + mountpoint = 'default' + type = 'icecast' + feeds_media_url = '' def __init__(self, station, q, logger, m3u): Thread.__init__(self) @@ -86,39 +97,31 @@ def __init__(self, station, q, logger, m3u): self.logger = logger self.m3u = m3u - if 'station_dir' in self.station: - self.station_dir = self.station['station_dir'] - - # Media - self.media_dir = self.station['media']['dir'] - self.media_format = self.station['media']['format'] - self.shuffle_mode = int(self.station['media']['shuffle']) - self.bitrate = self.station['media']['bitrate'] - self.ogg_quality = self.station['media']['ogg_quality'] - self.samplerate = self.station['media']['samplerate'] - self.voices = self.station['media']['voices'] - self.m3u_playlist_file = [] - if 'm3u' in self.station['media'].keys(): - self.m3u_playlist_file = self.station['media']['m3u'] - - # Server - if 'mountpoint' in self.station['server'].keys(): - self.mountpoint = self.station['server']['mountpoint'] - elif 'short_name' in self.station['infos'].keys(): - self.mountpoint = self.station['infos']['short_name'] - else: - self.mountpoint = 'default' + # Port the feeds key into the rss key if it exists (for backwards compatibility) + if 'feeds' in self.station.keys(): + self.station['rss'] = self.station['feeds'] + self.station.pop('feeds') + + # Get the mountpoint and short name first + if 'server' in self.station.keys(): + # Try to get the mountpoint as defined by the server.mountpoint parameter + self.mountpoint = str(self.station['server'].pop('mountpoint',self.mountpoint)) + self.appendtype = int(self.station['server'].pop('appendtype',self.appendtype)) + self.type = str(self.station['server'].pop('type',self.type)) + + # Get our media format since we need to to define a stream object + if 'media' in self.station.keys(): + self.media_format = str(self.station['media'].pop('format',self.media_format)) + + # We don't have a non-default mount point yet, so get info.short_name and try that + if self.mountpoint == 'default': + if 'infos' in self.station.keys(): + self.mountpoint = str(self.station['infos'].pop('short_name','default')) + # Set the short name based on the mount point. self.short_name = self.mountpoint - - if 'appendtype' in self.station['server'].keys(): - self.appendtype = self.station['server']['appendtype'] - - if 'type' in self.station['server']: - self.type = self.station['server']['type'] # 'icecast' | 'stream-m' - else: - self.type = 'icecast' - + + # Set up the stream channel if 'stream-m' in self.type: self.channel = HTTPStreamer() self.channel.mount = '/publish/' + self.mountpoint @@ -129,19 +132,161 @@ def __init__(self, station, q, logger, m3u): self.channel.mount = self.channel.mount + '.' + self.media_format else: sys.exit('Not a compatible server type. Choose "stream-m" or "icecast".') - - self.channel.url = self.station['infos']['url'] - self.channel.name = self.station['infos']['name'] - self.channel.genre = self.station['infos']['genre'] - self.channel.description = self.station['infos']['description'] + + # Apply defaults for the channel self.channel.format = self.media_format - self.channel.host = self.station['server']['host'] - self.channel.port = int(self.station['server']['port']) self.channel.user = 'source' - self.channel.password = self.station['server']['sourcepassword'] - self.channel.public = int(self.station['server']['public']) - self.channel.genre = self.station['infos']['genre'] - self.channel.description = self.station['infos']['description'] + + # Parse the rest of the key structure + for key in self.station.keys(): + if key == 'station_dir': + self.station_dir = self.station['station_dir'] + + if key == 'media': + # Media + for subkey in self.station[key].keys(): + if subkey == 'dir': + self.media_dir = str(self.station[key][subkey]) + + elif subkey == 'shuffle': + self.shuffle_mode = int(self.station[key][subkey]) + + elif subkey == 'bitrate': + self.bitrate = int(self.station[key][subkey]) + + elif subkey == 'ogg_quality': + self.ogg_quality = int(self.station[key][subkey]) + + elif subkey == 'samplerate': + self.samplerate = int(self.station[key][subkey]) + + elif subkey == 'voices': + self.voices = int(self.station[key][subkey]) + + elif subkey == 'm3u': + self.m3u_playlist_file = self.station[key][subkey] + + if key == 'infos': + # Stream Info + for subkey in self.station[key].keys(): + if subkey == 'url': + self.channel.url = str(self.station[key][subkey]) + + elif subkey == 'name': + self.channel.name = str(self.station[key][subkey]) + + elif subkey == 'genre': + self.channel.genre = str(self.station[key][subkey]) + + elif subkey == 'description': + self.channel.description = str(self.station[key][subkey]) + + if key == 'server': + # Server Info + for subkey in self.station[key].keys(): + if subkey == 'host': + self.channel.host = str(self.station[key][subkey]) + + elif subkey == 'port': + self.channel.port = str(self.station[key][subkey]) + + elif subkey == 'sourceuser': + self.channel.user = str(self.station[key][subkey]) + + elif subkey == 'sourcepassword': + self.channel.password = str(self.station[key][subkey]) + + elif subkey == 'public': + self.channel.public = str(self.station[key][subkey]) + + if key == 'rss': + # Server Info + for subkey in self.station[key].keys(): + if subkey == 'mode': + self.feeds_mode = int(self.station[key][subkey]) + + elif subkey == 'dir': + self.feeds_dir = str(self.station[key][subkey]) + + elif subkey == 'enclosure': + self.feeds_enclosure = int(self.station[key][subkey]) + + elif subkey == 'json': + self.feeds_json = int(self.station[key][subkey]) + + elif subkey == 'rss': + self.feeds_rss = int(self.station[key][subkey]) + + elif subkey == 'playlist': + self.feeds_playlist = int(self.station[key][subkey]) + + elif subkey == 'media_url': + self.feeds_media_url = str(self.station[key][subkey]) + + if key == 'control': + # Server Info + for subkey in self.station[key].keys(): + if subkey == 'mode': + self.osc_control_mode = int(self.station[key][subkey]) + + elif subkey == 'port': + self.osc_control_port = int(self.station[key][subkey]) + + if key == 'jingles': + # Server Info + for subkey in self.station[key].keys(): + if subkey == 'mode': + self.jingles_mode = int(self.station[key][subkey]) + + elif subkey == 'shuffle': + self.jingles_shuffle = int(self.station[key][subkey]) + + elif subkey == 'dir': + self.jingles_dir = str(self.station[key][subkey]) + + if key == 'relay': + # Server Info + for subkey in self.station[key].keys(): + if subkey == 'mode': + self.relay_mode = int(self.station[key][subkey]) + + elif subkey == 'url': + self.relay_url = str(self.station[key][subkey]) + + elif subkey == 'author': + self.relay_author = str(self.station[key][subkey]) + + if key == 'twitter': + # Server Info + for subkey in self.station[key].keys(): + if subkey == 'mode': + self.twitter_mode = int(self.station[key][subkey]) + + elif subkey == 'key': + self.twitter_key = str(self.station[key][subkey]) + + elif subkey == 'secret': + self.twitter_secret = str(self.station[key][subkey]) + + elif subkey == 'tags': + self.twitter_tags = str(self.station[key][subkey]).split(' ') + + elif subkey == 'messages': + self.twitter_messages = self.station[key][subkey] + if isinstance(self.twitter_messages, dict): + self.twitter_messages = list(self.twitter_messages) + + if key == 'record': + # Server Info + for subkey in self.station[key].keys(): + if subkey == 'mode': + self.record_mode = int(self.station[key][subkey]) + + elif subkey == 'dir': + self.record_dir = str(self.station[key][subkey]) + + + if self.channel.format == 'mp3': self.channel.audio_info = { 'bitrate': str(self.bitrate), 'samplerate': str(self.samplerate), @@ -156,29 +301,12 @@ def __init__(self, station, q, logger, m3u): self.channel_url = self.server_url + self.channel.mount # RSS - if 'feeds' in self.station: - self.station['rss'] = self.station['feeds'] - - if 'rss' in self.station: - if 'mode' in self.station['rss']: - self.feeds_mode = int(self.station['rss']['mode']) - self.feeds_dir = self.station['rss']['dir'] - self.feeds_enclosure = int(self.station['rss']['enclosure']) - if 'json' in self.station['rss']: - self.feeds_json = int(self.station['rss']['json']) - if 'rss' in self.station['rss']: - self.feeds_rss = int(self.station['rss']['rss']) - if 'playlist' in self.station['rss']: - self.feeds_playlist = int(self.station['rss']['playlist']) - + if self.feeds_media_url == '': self.feeds_media_url = self.channel.url + '/media/' - if 'media_url' in self.station['rss']: - if self.station['rss']['media_url']: - self.feeds_media_url = self.station['rss']['media_url'] self.base_name = self.feeds_dir + os.sep + self.short_name + '_' + self.channel.format - self.feeds_current_file = self.base_name + '_current.xml' - self.feeds_playlist_file = self.base_name + '_playlist.xml' + self.feeds_current_file = self.base_name + '_current' + self.feeds_playlist_file = self.base_name + '_playlist' # Logging self.logger.write_info('Opening ' + self.short_name + ' - ' + self.channel.name + \ @@ -194,60 +322,33 @@ def __init__(self, station, q, logger, m3u): self.player = Player(self.type) # OSCing - # mode = 0 means Off, mode = 1 means On - if 'control' in self.station: - self.osc_control_mode = int(self.station['control']['mode']) - if self.osc_control_mode: - self.osc_port = self.station['control']['port'] - self.osc_controller = OSCController(self.osc_port) - # OSC paths and callbacks - self.osc_controller.add_method('/media/next', 'i', self.media_next_callback) - self.osc_controller.add_method('/media/relay', 'i', self.relay_callback) - self.osc_controller.add_method('/twitter', 'i', self.twitter_callback) - self.osc_controller.add_method('/jingles', 'i', self.jingles_callback) - self.osc_controller.add_method('/record', 'i', self.record_callback) - self.osc_controller.add_method('/player', 'i', self.player_callback) - self.osc_controller.add_method('/run', 'i', self.run_callback) - self.osc_controller.start() + if self.osc_control_mode and self.osc_control_port: + self.osc_controller = OSCController(self.osc_control_port) + # OSC paths and callbacks + self.osc_controller.add_method('/media/next', 'i', self.media_next_callback) + self.osc_controller.add_method('/media/relay', 'i', self.relay_callback) + self.osc_controller.add_method('/twitter', 'i', self.twitter_callback) + self.osc_controller.add_method('/jingles', 'i', self.jingles_callback) + self.osc_controller.add_method('/record', 'i', self.record_callback) + self.osc_controller.add_method('/player', 'i', self.player_callback) + self.osc_controller.add_method('/run', 'i', self.run_callback) + self.osc_controller.start() # Jingling between each media. - if 'jingles' in self.station: - self.jingles_mode = int(self.station['jingles']['mode']) - self.jingles_shuffle = self.station['jingles']['shuffle'] - self.jingles_dir = self.station['jingles']['dir'] - if self.jingles_mode == 1: - self.jingles_callback('/jingles', [1]) + if self.jingles_mode: + self.jingles_callback('/jingles', [1]) # Relaying - if 'relay' in self.station: - self.relay_mode = int(self.station['relay']['mode']) - self.relay_url = self.station['relay']['url'] - self.relay_author = self.station['relay']['author'] - if self.relay_mode == 1: - self.relay_callback('/media/relay', [1]) + if self.relay_mode: + self.relay_callback('/media/relay', [1]) # Twitting - if 'twitter' in self.station: - self.twitter_mode = int(self.station['twitter']['mode']) - self.twitter_key = self.station['twitter']['key'] - self.twitter_secret = self.station['twitter']['secret'] - self.twitter_tags = self.station['twitter']['tags'].split(' ') - try: - self.twitter_messages = self.station['twitter']['message'] - if isinstance(self.twitter_messages, dict): - self.twitter_messages = list(self.twitter_messages) - except: - pass - - if self.twitter_mode: - self.twitter_callback('/twitter', [1]) + if self.twitter_mode: + self.twitter_callback('/twitter', [1]) # Recording - if 'record' in self.station: - self.record_mode = int(self.station['record']['mode']) - self.record_dir = self.station['record']['dir'] - if self.record_mode: - self.record_callback('/record', [1]) + if self.record_mode: + self.record_callback('/record', [1]) def run_callback(self, path, value): value = value[0] @@ -429,7 +530,7 @@ def get_next_media(self): new_tracks = new_playlist_set - playlist_set self.new_tracks = list(new_tracks.copy()) - if self.twitter_mode == 1 and self.counter: + if self.twitter_mode and self.counter: self.tweet() # Shake it, Fuzz it ! @@ -554,15 +655,12 @@ def update_feeds(self, media_list, rss_file, sub_title): items = rss_item_list,) if self.feeds_rss: - f = open(rss_file, 'w') + f = open(rss_file + '.xml', 'w') rss.write_xml(f, 'utf-8') f.close() if self.feeds_json: - path, fn = os.path.split(rss_file) - base, ext = os.path.splitext(fn) - json_file = os.path.join(self.feeds_dir, base + '.json') - f = open(json_file, 'w') + f = open(rss_file + '.json', 'w') f.write(json.dumps(json_data, separators=(',',':'))) f.close() From 25f92ca2481d9e63b0672783767e1665108abd7a Mon Sep 17 00:00:00 2001 From: achbed Date: Thu, 20 Nov 2014 23:33:47 -0600 Subject: [PATCH 24/82] Reverted a lot of changes Signed-off-by: achbed --- deefuzzer/station.py | 315 +++++++++++++++---------------------------- 1 file changed, 107 insertions(+), 208 deletions(-) diff --git a/deefuzzer/station.py b/deefuzzer/station.py index 20f0b75..fb88beb 100644 --- a/deefuzzer/station.py +++ b/deefuzzer/station.py @@ -67,16 +67,10 @@ class Station(Thread): lp = 1 player_mode = 0 osc_control_mode = 0 - osc_control_port = 16001 twitter_mode = 0 jingles_mode = 0 - jingles_shuffle = 1 - jingles_dir = '' relay_mode = 0 - relay_url = '' - relay_author = '' record_mode = 0 - record_dir = '' run_mode = 1 station_dir = None appendtype = 1 @@ -84,11 +78,6 @@ class Station(Thread): feeds_rss = 1 feeds_mode = 1 feeds_playlist = 1 - media_dir = '' - m3u_playlist_file = [] - mountpoint = 'default' - type = 'icecast' - feeds_media_url = '' def __init__(self, station, q, logger, m3u): Thread.__init__(self) @@ -97,31 +86,39 @@ def __init__(self, station, q, logger, m3u): self.logger = logger self.m3u = m3u - # Port the feeds key into the rss key if it exists (for backwards compatibility) - if 'feeds' in self.station.keys(): - self.station['rss'] = self.station['feeds'] - self.station.pop('feeds') - - # Get the mountpoint and short name first - if 'server' in self.station.keys(): - # Try to get the mountpoint as defined by the server.mountpoint parameter - self.mountpoint = str(self.station['server'].pop('mountpoint',self.mountpoint)) - self.appendtype = int(self.station['server'].pop('appendtype',self.appendtype)) - self.type = str(self.station['server'].pop('type',self.type)) - - # Get our media format since we need to to define a stream object - if 'media' in self.station.keys(): - self.media_format = str(self.station['media'].pop('format',self.media_format)) - - # We don't have a non-default mount point yet, so get info.short_name and try that - if self.mountpoint == 'default': - if 'infos' in self.station.keys(): - self.mountpoint = str(self.station['infos'].pop('short_name','default')) + if 'station_dir' in self.station: + self.station_dir = self.station['station_dir'] + + # Media + self.media_dir = self.station['media']['dir'] + self.media_format = self.station['media']['format'] + self.shuffle_mode = int(self.station['media']['shuffle']) + self.bitrate = self.station['media']['bitrate'] + self.ogg_quality = self.station['media']['ogg_quality'] + self.samplerate = self.station['media']['samplerate'] + self.voices = self.station['media']['voices'] + self.m3u_playlist_file = [] + if 'm3u' in self.station['media'].keys(): + self.m3u_playlist_file = self.station['media']['m3u'] + + # Server + if 'mountpoint' in self.station['server'].keys(): + self.mountpoint = self.station['server']['mountpoint'] + elif 'short_name' in self.station['infos'].keys(): + self.mountpoint = self.station['infos']['short_name'] + else: + self.mountpoint = 'default' - # Set the short name based on the mount point. self.short_name = self.mountpoint - - # Set up the stream channel + + if 'appendtype' in self.station['server'].keys(): + self.appendtype = int(self.station['server']['appendtype']) + + if 'type' in self.station['server']: + self.type = self.station['server']['type'] # 'icecast' | 'stream-m' + else: + self.type = 'icecast' + if 'stream-m' in self.type: self.channel = HTTPStreamer() self.channel.mount = '/publish/' + self.mountpoint @@ -132,161 +129,19 @@ def __init__(self, station, q, logger, m3u): self.channel.mount = self.channel.mount + '.' + self.media_format else: sys.exit('Not a compatible server type. Choose "stream-m" or "icecast".') - - # Apply defaults for the channel + + self.channel.url = self.station['infos']['url'] + self.channel.name = self.station['infos']['name'] + self.channel.genre = self.station['infos']['genre'] + self.channel.description = self.station['infos']['description'] self.channel.format = self.media_format + self.channel.host = self.station['server']['host'] + self.channel.port = int(self.station['server']['port']) self.channel.user = 'source' - - # Parse the rest of the key structure - for key in self.station.keys(): - if key == 'station_dir': - self.station_dir = self.station['station_dir'] - - if key == 'media': - # Media - for subkey in self.station[key].keys(): - if subkey == 'dir': - self.media_dir = str(self.station[key][subkey]) - - elif subkey == 'shuffle': - self.shuffle_mode = int(self.station[key][subkey]) - - elif subkey == 'bitrate': - self.bitrate = int(self.station[key][subkey]) - - elif subkey == 'ogg_quality': - self.ogg_quality = int(self.station[key][subkey]) - - elif subkey == 'samplerate': - self.samplerate = int(self.station[key][subkey]) - - elif subkey == 'voices': - self.voices = int(self.station[key][subkey]) - - elif subkey == 'm3u': - self.m3u_playlist_file = self.station[key][subkey] - - if key == 'infos': - # Stream Info - for subkey in self.station[key].keys(): - if subkey == 'url': - self.channel.url = str(self.station[key][subkey]) - - elif subkey == 'name': - self.channel.name = str(self.station[key][subkey]) - - elif subkey == 'genre': - self.channel.genre = str(self.station[key][subkey]) - - elif subkey == 'description': - self.channel.description = str(self.station[key][subkey]) - - if key == 'server': - # Server Info - for subkey in self.station[key].keys(): - if subkey == 'host': - self.channel.host = str(self.station[key][subkey]) - - elif subkey == 'port': - self.channel.port = str(self.station[key][subkey]) - - elif subkey == 'sourceuser': - self.channel.user = str(self.station[key][subkey]) - - elif subkey == 'sourcepassword': - self.channel.password = str(self.station[key][subkey]) - - elif subkey == 'public': - self.channel.public = str(self.station[key][subkey]) - - if key == 'rss': - # Server Info - for subkey in self.station[key].keys(): - if subkey == 'mode': - self.feeds_mode = int(self.station[key][subkey]) - - elif subkey == 'dir': - self.feeds_dir = str(self.station[key][subkey]) - - elif subkey == 'enclosure': - self.feeds_enclosure = int(self.station[key][subkey]) - - elif subkey == 'json': - self.feeds_json = int(self.station[key][subkey]) - - elif subkey == 'rss': - self.feeds_rss = int(self.station[key][subkey]) - - elif subkey == 'playlist': - self.feeds_playlist = int(self.station[key][subkey]) - - elif subkey == 'media_url': - self.feeds_media_url = str(self.station[key][subkey]) - - if key == 'control': - # Server Info - for subkey in self.station[key].keys(): - if subkey == 'mode': - self.osc_control_mode = int(self.station[key][subkey]) - - elif subkey == 'port': - self.osc_control_port = int(self.station[key][subkey]) - - if key == 'jingles': - # Server Info - for subkey in self.station[key].keys(): - if subkey == 'mode': - self.jingles_mode = int(self.station[key][subkey]) - - elif subkey == 'shuffle': - self.jingles_shuffle = int(self.station[key][subkey]) - - elif subkey == 'dir': - self.jingles_dir = str(self.station[key][subkey]) - - if key == 'relay': - # Server Info - for subkey in self.station[key].keys(): - if subkey == 'mode': - self.relay_mode = int(self.station[key][subkey]) - - elif subkey == 'url': - self.relay_url = str(self.station[key][subkey]) - - elif subkey == 'author': - self.relay_author = str(self.station[key][subkey]) - - if key == 'twitter': - # Server Info - for subkey in self.station[key].keys(): - if subkey == 'mode': - self.twitter_mode = int(self.station[key][subkey]) - - elif subkey == 'key': - self.twitter_key = str(self.station[key][subkey]) - - elif subkey == 'secret': - self.twitter_secret = str(self.station[key][subkey]) - - elif subkey == 'tags': - self.twitter_tags = str(self.station[key][subkey]).split(' ') - - elif subkey == 'messages': - self.twitter_messages = self.station[key][subkey] - if isinstance(self.twitter_messages, dict): - self.twitter_messages = list(self.twitter_messages) - - if key == 'record': - # Server Info - for subkey in self.station[key].keys(): - if subkey == 'mode': - self.record_mode = int(self.station[key][subkey]) - - elif subkey == 'dir': - self.record_dir = str(self.station[key][subkey]) - - - + self.channel.password = self.station['server']['sourcepassword'] + self.channel.public = int(self.station['server']['public']) + self.channel.genre = self.station['infos']['genre'] + self.channel.description = self.station['infos']['description'] if self.channel.format == 'mp3': self.channel.audio_info = { 'bitrate': str(self.bitrate), 'samplerate': str(self.samplerate), @@ -301,8 +156,25 @@ def __init__(self, station, q, logger, m3u): self.channel_url = self.server_url + self.channel.mount # RSS - if self.feeds_media_url == '': + if 'feeds' in self.station: + self.station['rss'] = self.station['feeds'] + + if 'rss' in self.station: + if 'mode' in self.station['rss']: + self.feeds_mode = int(self.station['rss']['mode']) + self.feeds_dir = self.station['rss']['dir'] + self.feeds_enclosure = int(self.station['rss']['enclosure']) + if 'json' in self.station['rss']: + self.feeds_json = int(self.station['rss']['json']) + if 'rss' in self.station['rss']: + self.feeds_rss = int(self.station['rss']['rss']) + if 'playlist' in self.station['rss']: + self.feeds_playlist = int(self.station['rss']['playlist']) + self.feeds_media_url = self.channel.url + '/media/' + if 'media_url' in self.station['rss']: + if self.station['rss']['media_url']: + self.feeds_media_url = self.station['rss']['media_url'] self.base_name = self.feeds_dir + os.sep + self.short_name + '_' + self.channel.format self.feeds_current_file = self.base_name + '_current' @@ -322,33 +194,60 @@ def __init__(self, station, q, logger, m3u): self.player = Player(self.type) # OSCing - if self.osc_control_mode and self.osc_control_port: - self.osc_controller = OSCController(self.osc_control_port) - # OSC paths and callbacks - self.osc_controller.add_method('/media/next', 'i', self.media_next_callback) - self.osc_controller.add_method('/media/relay', 'i', self.relay_callback) - self.osc_controller.add_method('/twitter', 'i', self.twitter_callback) - self.osc_controller.add_method('/jingles', 'i', self.jingles_callback) - self.osc_controller.add_method('/record', 'i', self.record_callback) - self.osc_controller.add_method('/player', 'i', self.player_callback) - self.osc_controller.add_method('/run', 'i', self.run_callback) - self.osc_controller.start() + # mode = 0 means Off, mode = 1 means On + if 'control' in self.station: + self.osc_control_mode = int(self.station['control']['mode']) + if self.osc_control_mode: + self.osc_port = self.station['control']['port'] + self.osc_controller = OSCController(self.osc_port) + # OSC paths and callbacks + self.osc_controller.add_method('/media/next', 'i', self.media_next_callback) + self.osc_controller.add_method('/media/relay', 'i', self.relay_callback) + self.osc_controller.add_method('/twitter', 'i', self.twitter_callback) + self.osc_controller.add_method('/jingles', 'i', self.jingles_callback) + self.osc_controller.add_method('/record', 'i', self.record_callback) + self.osc_controller.add_method('/player', 'i', self.player_callback) + self.osc_controller.add_method('/run', 'i', self.run_callback) + self.osc_controller.start() # Jingling between each media. - if self.jingles_mode: - self.jingles_callback('/jingles', [1]) + if 'jingles' in self.station: + self.jingles_mode = int(self.station['jingles']['mode']) + self.jingles_shuffle = self.station['jingles']['shuffle'] + self.jingles_dir = self.station['jingles']['dir'] + if self.jingles_mode == 1: + self.jingles_callback('/jingles', [1]) # Relaying - if self.relay_mode: - self.relay_callback('/media/relay', [1]) + if 'relay' in self.station: + self.relay_mode = int(self.station['relay']['mode']) + self.relay_url = self.station['relay']['url'] + self.relay_author = self.station['relay']['author'] + if self.relay_mode == 1: + self.relay_callback('/media/relay', [1]) # Twitting - if self.twitter_mode: - self.twitter_callback('/twitter', [1]) + if 'twitter' in self.station: + self.twitter_mode = int(self.station['twitter']['mode']) + self.twitter_key = self.station['twitter']['key'] + self.twitter_secret = self.station['twitter']['secret'] + self.twitter_tags = self.station['twitter']['tags'].split(' ') + try: + self.twitter_messages = self.station['twitter']['message'] + if isinstance(self.twitter_messages, dict): + self.twitter_messages = list(self.twitter_messages) + except: + pass + + if self.twitter_mode: + self.twitter_callback('/twitter', [1]) # Recording - if self.record_mode: - self.record_callback('/record', [1]) + if 'record' in self.station: + self.record_mode = int(self.station['record']['mode']) + self.record_dir = self.station['record']['dir'] + if self.record_mode: + self.record_callback('/record', [1]) def run_callback(self, path, value): value = value[0] @@ -530,7 +429,7 @@ def get_next_media(self): new_tracks = new_playlist_set - playlist_set self.new_tracks = list(new_tracks.copy()) - if self.twitter_mode and self.counter: + if self.twitter_mode == 1 and self.counter: self.tweet() # Shake it, Fuzz it ! From 6af9ef5363388bbfa70cc20618225f02df15e946 Mon Sep 17 00:00:00 2001 From: achbed Date: Thu, 20 Nov 2014 23:41:10 -0600 Subject: [PATCH 25/82] Ensure that integer parameters are treated properly Removed double-set options Signed-off-by: achbed --- deefuzzer/station.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/deefuzzer/station.py b/deefuzzer/station.py index fb88beb..ef4de21 100644 --- a/deefuzzer/station.py +++ b/deefuzzer/station.py @@ -93,10 +93,10 @@ def __init__(self, station, q, logger, m3u): self.media_dir = self.station['media']['dir'] self.media_format = self.station['media']['format'] self.shuffle_mode = int(self.station['media']['shuffle']) - self.bitrate = self.station['media']['bitrate'] - self.ogg_quality = self.station['media']['ogg_quality'] - self.samplerate = self.station['media']['samplerate'] - self.voices = self.station['media']['voices'] + self.bitrate = int(self.station['media']['bitrate']) + self.ogg_quality = int(self.station['media']['ogg_quality']) + self.samplerate = int(self.station['media']['samplerate']) + self.voices = int(self.station['media']['voices']) self.m3u_playlist_file = [] if 'm3u' in self.station['media'].keys(): self.m3u_playlist_file = self.station['media']['m3u'] @@ -140,8 +140,6 @@ def __init__(self, station, q, logger, m3u): self.channel.user = 'source' self.channel.password = self.station['server']['sourcepassword'] self.channel.public = int(self.station['server']['public']) - self.channel.genre = self.station['infos']['genre'] - self.channel.description = self.station['infos']['description'] if self.channel.format == 'mp3': self.channel.audio_info = { 'bitrate': str(self.bitrate), 'samplerate': str(self.samplerate), @@ -173,7 +171,7 @@ def __init__(self, station, q, logger, m3u): self.feeds_media_url = self.channel.url + '/media/' if 'media_url' in self.station['rss']: - if self.station['rss']['media_url']: + if not self.station['rss']['media_url'] == '': self.feeds_media_url = self.station['rss']['media_url'] self.base_name = self.feeds_dir + os.sep + self.short_name + '_' + self.channel.format @@ -198,7 +196,7 @@ def __init__(self, station, q, logger, m3u): if 'control' in self.station: self.osc_control_mode = int(self.station['control']['mode']) if self.osc_control_mode: - self.osc_port = self.station['control']['port'] + self.osc_port = int(self.station['control']['port']) self.osc_controller = OSCController(self.osc_port) # OSC paths and callbacks self.osc_controller.add_method('/media/next', 'i', self.media_next_callback) @@ -213,7 +211,7 @@ def __init__(self, station, q, logger, m3u): # Jingling between each media. if 'jingles' in self.station: self.jingles_mode = int(self.station['jingles']['mode']) - self.jingles_shuffle = self.station['jingles']['shuffle'] + self.jingles_shuffle = int(self.station['jingles']['shuffle']) self.jingles_dir = self.station['jingles']['dir'] if self.jingles_mode == 1: self.jingles_callback('/jingles', [1]) From c045f7ac834228e26e262f5c626fb67932bc7f99 Mon Sep 17 00:00:00 2001 From: achbed Date: Thu, 20 Nov 2014 23:42:35 -0600 Subject: [PATCH 26/82] Fixed a few issues with setting up logging Signed-off-by: achbed --- deefuzzer/core.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/deefuzzer/core.py b/deefuzzer/core.py index 9f401c5..af027f4 100644 --- a/deefuzzer/core.py +++ b/deefuzzer/core.py @@ -62,11 +62,14 @@ def __init__(self, conf_file): self.conf_file = conf_file self.conf = get_conf_dict(self.conf_file) + if not 'deefuzzer' in self.conf.keys(): + return + # Get the log setting first (if possible) - log_file = str(self.conf.pop('log', '')) + log_file = str(self.conf['deefuzzer'].pop('log', '')) log_dir = os.sep.join(log_file.split(os.sep)[:-1]) if not os.path.exists(log_dir) and log_dir: - os.makedirs(m3u_dir) + os.makedirs(log_dir) self.logger = Logger(log_file) for key in self.conf['deefuzzer'].keys(): From 3494fb1e280519fdfb934e61e7214901a05c4112 Mon Sep 17 00:00:00 2001 From: achbed Date: Fri, 21 Nov 2014 00:04:59 -0600 Subject: [PATCH 27/82] Added some logging when a station fails to initialize Signed-off-by: achbed --- deefuzzer/core.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/deefuzzer/core.py b/deefuzzer/core.py index af027f4..9e09987 100644 --- a/deefuzzer/core.py +++ b/deefuzzer/core.py @@ -210,6 +210,11 @@ def run(self): station = merge_defaults(station, self.conf['deefuzzer']['stationdefaults']) self.stations.append(Station(station, q, self.logger, self.m3u)) except Exception: + name = str(i) + if 'info' in station.keys(): + if 'short_name' in station['infos']: + name = station['infos']['short_name'] + self.logger.write_error('Error starting station ' + name) continue if self.m3u: From d5fb888cf1919ecd5c818816dbd4a00da31dc8e5 Mon Sep 17 00:00:00 2001 From: achbed Date: Fri, 21 Nov 2014 22:57:24 -0600 Subject: [PATCH 28/82] Added new station.feeds.showfilename option that includes the media file name in the feed Added new station.feeds.showfilepath option that includes the server path to the media file in the feed Signed-off-by: achbed --- deefuzzer/station.py | 18 +++++++++++++++--- example/deefuzzer_doc.xml | 4 ++++ 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/deefuzzer/station.py b/deefuzzer/station.py index ef4de21..9063185 100644 --- a/deefuzzer/station.py +++ b/deefuzzer/station.py @@ -78,6 +78,8 @@ class Station(Thread): feeds_rss = 1 feeds_mode = 1 feeds_playlist = 1 + feeds_showfilepath = 0 + feeds_showfilename = 0 def __init__(self, station, q, logger, m3u): Thread.__init__(self) @@ -168,6 +170,10 @@ def __init__(self, station, q, logger, m3u): self.feeds_rss = int(self.station['rss']['rss']) if 'playlist' in self.station['rss']: self.feeds_playlist = int(self.station['rss']['playlist']) + if 'showfilename' in self.station['rss']: + self.feeds_showfilename = int(self.station['rss']['showfilename']) + if 'showfilepath' in self.station['rss']: + self.feeds_showfilepath = int(self.station['rss']['showfilepath']) self.feeds_media_url = self.channel.url + '/media/' if 'media_url' in self.station['rss']: @@ -464,19 +470,24 @@ def media_to_objs(self, media_list): file_name, file_title, file_ext = get_file_info(media) if file_ext.lower() == 'mp3' or mimetypes.guess_type(media)[0] == 'audio/mpeg': try: - media_objs.append(Mp3(media)) + file_meta = Mp3(media) except: continue elif file_ext.lower() == 'ogg' or mimetypes.guess_type(media)[0] == 'audio/ogg': try: - media_objs.append(Ogg(media)) + file_meta = Ogg(media) except: continue elif file_ext.lower() == 'webm' or mimetypes.guess_type(media)[0] == 'video/webm': try: - media_objs.append(WebM(media)) + file_meta = WebM(media) except: continue + if self.feeds_showfilename: + file_meta.metadata['filename'] = file_name.decode( "utf-8" ) #decode needed for some weird filenames + if self.feeds_showfilepath: + file_meta.metadata['filepath'] = media.decode( "utf-8" ) #decode needed for some weird filenames + media_objs.append(file_meta) return media_objs def update_feeds(self, media_list, rss_file, sub_title): @@ -503,6 +514,7 @@ def update_feeds(self, media_list, rss_file, sub_title): media_description = '' media_description_item = '' + for key in media.metadata.keys(): if media.metadata[key] != '': media_description += media_description_item % (key.capitalize(), diff --git a/example/deefuzzer_doc.xml b/example/deefuzzer_doc.xml index fd848dd..138a6a9 100644 --- a/example/deefuzzer_doc.xml +++ b/example/deefuzzer_doc.xml @@ -110,6 +110,10 @@ http://localhost/media/ + + 1 + + 0 From d2f5d592c84f2f1602d1bec4b9541a8487a89728 Mon Sep 17 00:00:00 2001 From: achbed Date: Fri, 28 Nov 2014 14:27:22 -0600 Subject: [PATCH 29/82] Refactored main icecast loop for readability Added additional error handling in main icecast loop Added additional error handling to icecast chunk send loop Signed-off-by: achbed --- deefuzzer/station.py | 168 ++++++++++++++++++++++++------------------- 1 file changed, 93 insertions(+), 75 deletions(-) diff --git a/deefuzzer/station.py b/deefuzzer/station.py index 9063185..45973a3 100644 --- a/deefuzzer/station.py +++ b/deefuzzer/station.py @@ -664,6 +664,33 @@ def ping_server(self): self.q.task_done() pass + def icecastloop_nextmedia(self): + self.q.get(1) + self.next_media = 0 + self.media = self.get_next_media() + self.counter += 1 + if self.relay_mode: + self.set_relay_mode() + elif os.path.exists(self.media) and not os.sep+'.' in self.media: + if self.lp == 0: + self.logger.write_error('Station ' + self.channel_url + \ + ' : has no media to stream !') + return false + self.set_read_mode() + self.q.task_done() + + return true + + def icecastloop_metadata(self): + self.q.get(1) + if (not (self.jingles_mode and (self.counter % 2)) or \ + self.relay_mode) and self.twitter_mode: + self.update_twitter_current() + self.channel.set_metadata({'song': self.song, 'charset': 'utf-8'}) + self.q.task_done() + return true + + def run(self): self.q.get(1) self.ping_server() @@ -687,86 +714,77 @@ def run(self): self.q.task_done() while self.run_mode: - self.q.get(1) - self.next_media = 0 - self.media = self.get_next_media() - self.counter += 1 - if self.relay_mode: - self.set_relay_mode() - elif os.path.exists(self.media) and not os.sep+'.' in self.media: - if self.lp == 0: - self.logger.write_error('Station ' + self.channel_url + \ - ' : has no media to stream !') - break - self.set_read_mode() - self.q.task_done() - - self.q.get(1) - if (not (self.jingles_mode and (self.counter % 2)) or \ - self.relay_mode) and self.twitter_mode: - try: - self.update_twitter_current() - except: - continue try: - self.channel.set_metadata({'song': self.song, 'charset': 'utf-8',}) - except: - continue - self.q.task_done() - - - for self.chunk in self.stream: - if self.next_media or not self.run_mode: + if not self.icecastloop_nextmedia(): break - if self.record_mode: - try: - self.q.get(1) - self.recorder.write(self.chunk) - self.q.task_done() - except: - self.logger.write_error('Station ' + \ - self.channel_url + \ - ' : could not write the buffer to the file') - self.q.task_done() - continue - try: - self.q.get(1) - self.channel.send(self.chunk) - self.channel.sync() - self.q.task_done() - except: - self.logger.write_error('Station ' + \ - self.channel_url + \ - ' : could not send the buffer') - self.q.task_done() + if not self.icecastloop_metadata(): + break + + for self.chunk in self.stream: try: - self.q.get(1) - self.channel.close() - self.logger.write_info('Station ' + \ - self.channel_url + \ - ' : channel closed') - self.q.task_done() - except: - self.logger.write_error('Station ' + \ + if self.next_media or not self.run_mode: + break + + if self.record_mode: + try: + self.q.get(1) + self.recorder.write(self.chunk) + self.q.task_done() + except: + self.logger.write_error('Station ' + \ self.channel_url + \ - ' : could not close the channel') - self.q.task_done() - continue - try: - self.ping_server() - self.q.get(1) - self.channel_open() - self.channel.set_metadata({'song': self.song, 'charset': 'utf8',}) - self.logger.write_info('Station ' + self.channel_url + \ - ' : channel restarted') - self.q.task_done() - except: - self.logger.write_error('Station ' + self.channel_url + \ - ' : could not restart the channel') - self.q.task_done() + ' : could not write the buffer to the file') + self.q.task_done() + continue + + try: + self.q.get(1) + self.channel.send(self.chunk) + self.channel.sync() + self.q.task_done() + except: + self.logger.write_error('Station ' + \ + self.channel_url + \ + ' : could not send the buffer') + self.q.task_done() + + try: + self.q.get(1) + self.channel.close() + self.logger.write_info('Station ' + \ + self.channel_url + \ + ' : channel closed') + self.q.task_done() + except: + self.logger.write_error('Station ' + \ + self.channel_url + \ + ' : could not close the channel') + self.q.task_done() + continue + + try: + self.ping_server() + self.q.get(1) + self.channel_open() + self.channel.set_metadata({'song': self.song, 'charset': 'utf8',}) + self.logger.write_info('Station ' + self.channel_url + \ + ' : channel restarted') + self.q.task_done() + except: + self.logger.write_error('Station ' + self.channel_url + \ + ' : could not restart the channel') + self.q.task_done() + continue + continue + + except: # send chunk loop exception continue - continue - + # send chunk loop end + + except: # while run_mode exception + continue + # while run_mode loop end + if self.record_mode: self.recorder.close() From ec26e790defff7ce3e7fd63ee9a65eecddc7e91f Mon Sep 17 00:00:00 2001 From: achbed Date: Fri, 28 Nov 2014 22:27:55 -0600 Subject: [PATCH 30/82] Removed extraneous "pass"es Standardized error and info logging within station class Fixed typo on truthy values --- deefuzzer/station.py | 108 ++++++++++++++++++------------------------- 1 file changed, 46 insertions(+), 62 deletions(-) diff --git a/deefuzzer/station.py b/deefuzzer/station.py index 45973a3..0397383 100644 --- a/deefuzzer/station.py +++ b/deefuzzer/station.py @@ -185,7 +185,7 @@ def __init__(self, station, q, logger, m3u): self.feeds_playlist_file = self.base_name + '_playlist' # Logging - self.logger.write_info('Opening ' + self.short_name + ' - ' + self.channel.name + \ + self._info('Opening ' + self.short_name + ' - ' + self.channel.name + \ ' (' + str(self.lp) + ' tracks)...') self.metadata_relative_dir = 'metadata' @@ -253,19 +253,23 @@ def __init__(self, station, q, logger, m3u): if self.record_mode: self.record_callback('/record', [1]) + def _info(self, msg): + self.logger.write_info('Station ' + self.channel_url + ': ' + str(msg)) + + def _err(self, msg): + self.logger.write_error('Station ' + self.channel_url + ': ' + str(msg)) + def run_callback(self, path, value): value = value[0] self.run_mode = value - message = "Station " + self.channel_url + \ - " : received OSC message '%s' with arguments '%d'" % (path, value) - self.logger.write_info(message) + message = "received OSC message '%s' with arguments '%d'" % (path, value) + self._info(message) def media_next_callback(self, path, value): value = value[0] self.next_media = value - message = "Station " + self.channel_url + \ - " : received OSC message '%s' with arguments '%d'" % (path, value) - self.logger.write_info(message) + message = "received OSC message '%s' with arguments '%d'" % (path, value) + self._info(message) def relay_callback(self, path, value): value = value[0] @@ -280,22 +284,19 @@ def relay_callback(self, path, value): self.id = 0 self.next_media = 1 - message = "Station " + self.channel_url + \ - " : received OSC message '%s' with arguments '%d'" % (path, value) - self.logger.write_info(message) - message = "Station " + self.channel_url + \ - " : relaying : %s" % self.relay_url - self.logger.write_info(message) + message = "received OSC message '%s' with arguments '%d'" % (path, value) + self._info(message) + message = "relaying : %s" % self.relay_url + self._info(message) def twitter_callback(self, path, value): value = value[0] self.twitter = Twitter(self.twitter_key, self.twitter_secret) self.twitter_mode = value - message = "Station " + self.channel_url + \ - " : received OSC message '%s' with arguments '%d'" % (path, value) + message = "received OSC message '%s' with arguments '%d'" % (path, value) self.m3u_url = self.channel.url + '/m3u/' + self.m3u.split(os.sep)[-1] self.feeds_url = self.channel.url + '/rss/' + self.feeds_playlist_file.split(os.sep)[-1] - self.logger.write_info(message) + self._info(message) def jingles_callback(self, path, value): value = value[0] @@ -304,9 +305,8 @@ def jingles_callback(self, path, value): self.jingles_length = len(self.jingles_list) self.jingle_id = 0 self.jingles_mode = value - message = "Station " + self.channel_url + \ - " : received OSC message '%s' with arguments '%d'" % (path, value) - self.logger.write_info(message) + message = "received OSC message '%s' with arguments '%d'" % (path, value) + self._info(message) def record_callback(self, path, value): value = value[0] @@ -324,6 +324,7 @@ def record_callback(self, path, value): self.recorder.close() except: pass + if self.type == 'icecast': date = datetime.datetime.now().strftime("%Y") if self.channel.format == 'mp3': @@ -338,16 +339,14 @@ def record_callback(self, path, value): media.write_tags() self.record_mode = value - message = "Station " + self.channel_url + \ - " : received OSC message '%s' with arguments '%d'" % (path, value) - self.logger.write_info(message) + message = "received OSC message '%s' with arguments '%d'" % (path, value) + self._info(message) def player_callback(self, path, value): value = value[0] self.player_mode = value - message = "Station " + self.channel_url + \ - " : received OSC message '%s' with arguments '%d'" % (path, value) - self.logger.write_info(message) + message = "received OSC message '%s' with arguments '%d'" % (path, value) + self._info(message) def get_playlist(self): file_list = [] @@ -417,8 +416,7 @@ def get_next_media(self): if self.shuffle_mode: random.shuffle(self.playlist) - self.logger.write_info('Station ' + self.channel_url + \ - ' : generating new playlist (' + str(self.lp) + ' tracks)') + self._info('generating new playlist (' + str(self.lp) + ' tracks)') if self.feeds_playlist: self.update_feeds(self.media_to_objs(self.playlist), self.feeds_playlist_file, '(playlist)') @@ -446,8 +444,7 @@ def get_next_media(self): self.playlist = playlist - self.logger.write_info('Station ' + self.channel_url + \ - ' : generating new playlist (' + str(self.lp) + ' tracks)') + self._info('generating new playlist (' + str(self.lp) + ' tracks)') if self.feeds_playlist: self.update_feeds(self.media_to_objs(self.playlist), self.feeds_playlist_file, '(playlist)') @@ -461,7 +458,7 @@ def get_next_media(self): return media else: mess = 'No media in media_dir !' - self.logger.write_error(mess) + self._err(mess) sys.exit(mess) def media_to_objs(self, media_list): @@ -576,10 +573,9 @@ def update_feeds(self, media_list, rss_file, sub_title): def update_twitter(self, message): try: self.twitter.post(message.decode('utf8')) - self.logger.write_info('Twitting : "' + message + '"') + self._info('Twitting : "' + message + '"') except: - self.logger.write_error('Twitting : "' + message + '"') - pass + self._err('Twitting : "' + message + '"') def set_relay_mode(self): self.prefix = '#nowplaying #LIVE' @@ -619,8 +615,8 @@ def set_read_mode(self): self.artist = self.artist.encode('utf-8') self.metadata_file = self.metadata_dir + os.sep + self.current_media_obj[0].file_name + '.xml' self.update_feeds(self.current_media_obj, self.feeds_current_file, '(currently playing)') - self.logger.write_info('DeeFuzzing on %s : id = %s, name = %s' \ - % (self.short_name, self.id, self.current_media_obj[0].file_name)) + self._info('DeeFuzzing: id = %s, name = %s' \ + % (self.id, self.current_media_obj[0].file_name)) self.player.set_media(self.media) if self.player_mode: @@ -652,17 +648,16 @@ def ping_server(self): self.q.get(1) server = urllib.urlopen(self.server_url) self.server_ping = True - text = 'Station ' + self.channel_url + ' : channel available' - self.logger.write_info(text) + text = 'channel available' + self._info(text) self.q.task_done() except: time.sleep(1) if log: - text = 'Station ' + self.channel_url + ' : could not connect the channel' - self.logger.write_error(text) + text = 'could not connect the channel' + self._err(text) log = False self.q.task_done() - pass def icecastloop_nextmedia(self): self.q.get(1) @@ -673,13 +668,12 @@ def icecastloop_nextmedia(self): self.set_relay_mode() elif os.path.exists(self.media) and not os.sep+'.' in self.media: if self.lp == 0: - self.logger.write_error('Station ' + self.channel_url + \ - ' : has no media to stream !') - return false + self._err('has no media to stream !') + return False self.set_read_mode() self.q.task_done() - return true + return True def icecastloop_metadata(self): self.q.get(1) @@ -688,7 +682,7 @@ def icecastloop_metadata(self): self.update_twitter_current() self.channel.set_metadata({'song': self.song, 'charset': 'utf-8'}) self.q.task_done() - return true + return True def run(self): @@ -710,7 +704,7 @@ def run(self): if self.type == 'icecast': self.q.get(1) self.channel_open() - self.logger.write_info('Station ' + self.channel_url + ' : channel connected') + self._info('channel connected') self.q.task_done() while self.run_mode: @@ -731,9 +725,7 @@ def run(self): self.recorder.write(self.chunk) self.q.task_done() except: - self.logger.write_error('Station ' + \ - self.channel_url + \ - ' : could not write the buffer to the file') + self._err('could not write the buffer to the file') self.q.task_done() continue @@ -743,22 +735,16 @@ def run(self): self.channel.sync() self.q.task_done() except: - self.logger.write_error('Station ' + \ - self.channel_url + \ - ' : could not send the buffer') + self._err('could not send the buffer') self.q.task_done() try: self.q.get(1) self.channel.close() - self.logger.write_info('Station ' + \ - self.channel_url + \ - ' : channel closed') + self._info('channel closed') self.q.task_done() except: - self.logger.write_error('Station ' + \ - self.channel_url + \ - ' : could not close the channel') + self._err('could not close the channel') self.q.task_done() continue @@ -767,12 +753,10 @@ def run(self): self.q.get(1) self.channel_open() self.channel.set_metadata({'song': self.song, 'charset': 'utf8',}) - self.logger.write_info('Station ' + self.channel_url + \ - ' : channel restarted') + self._info('channel restarted') self.q.task_done() except: - self.logger.write_error('Station ' + self.channel_url + \ - ' : could not restart the channel') + self._err('could not restart the channel') self.q.task_done() continue continue From 84298a1c345124da613e501c0d0d47c5f3d0c912 Mon Sep 17 00:00:00 2001 From: achbed Date: Fri, 28 Nov 2014 22:47:34 -0600 Subject: [PATCH 31/82] Added init.d and default config scripts to scripts folder (sets up deefuzzer as a system-level daemon) Signed-off-by: achbed --- scripts/etc/default/deefuzzer | 22 +++++++++++ scripts/etc/init.d/deefuzzer | 74 +++++++++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+) create mode 100644 scripts/etc/default/deefuzzer create mode 100644 scripts/etc/init.d/deefuzzer diff --git a/scripts/etc/default/deefuzzer b/scripts/etc/default/deefuzzer new file mode 100644 index 0000000..4b0a59d --- /dev/null +++ b/scripts/etc/default/deefuzzer @@ -0,0 +1,22 @@ +# Defaults for deefuzzer initscript +# sourced by /etc/init.d/deefuzzer +# installed at /etc/default/deefuzzer by the maintainer scripts + +# +# This is a POSIX shell fragment +# + +# Full path to the server configuration file +CONFIGFILE="/srv/pt/patestapes.com/deefuzzer.xml" + +# Name or ID of the user and group the daemon should run under +USERID=deefuzzer +GROUPID=staff + +# Edit /etc/deefuzzer.xml to match your environment. +# Change this to true when done to enable the init.d script +ENABLE=false + +# Uncomment below to make the service startup more verbose +QUIET="--verbose" + diff --git a/scripts/etc/init.d/deefuzzer b/scripts/etc/init.d/deefuzzer new file mode 100644 index 0000000..703b89a --- /dev/null +++ b/scripts/etc/init.d/deefuzzer @@ -0,0 +1,74 @@ +#! /bin/sh +### BEGIN INIT INFO +# Provides: deefuzzer +# Required-Start: $remote_fs $network +# Required-Stop: $remote_fs +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: Starts the deefuzzer streaming player +### END INIT INFO +# +# deefuzzer +# +# Written By Dennis Wallace (github@achbed.org) +# Based on code by Miquel van Smoorenburg . +# Modified for Debian +# by Ian Murdock . +# +# Further modified by Keegan Quinn +# for use with Icecast 2 +# + +PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin +DAEMON=/usr/local/bin/deefuzzer +NAME=deefuzzer +DESC=deefuzzer + +test -x $DAEMON || exit 0 + +. /lib/lsb/init-functions + +# Defaults +CONFIGFILE="/etc/deefuzzer/deefuzzer.xml" +CONFIGDEFAULTFILE="/etc/default/deefuzzer" +USERID=deefuzzer +GROUPID=staff +ENABLE="false" +QUIET="--quiet" + +# Reads config file (will override defaults above) +[ -r "$CONFIGDEFAULTFILE" ] && . $CONFIGDEFAULTFILE + +if [ "$ENABLE" != "true" ]; then + echo "$NAME daemon disabled - read $CONFIGDEFAULTFILE." + exit 0 +fi + +set -e + +case "$1" in + start) + echo -n "Starting $DESC: " + $DAEMON $CONFIGFILE & + echo "$NAME." + ;; + stop) + echo -n "Stopping $DESC: " + # Send TERM after 5 seconds, wait at most 30 seconds. + killall $NAME + echo "$NAME." + ;; + reload|force-reload|restart) + echo -n "Restarting $DESC: " + # Send TERM after 5 seconds, wait at most 30 seconds. + killall deefuzzer + $DAEMON $CONFIGFILE & + echo "$NAME." + ;; + *) + echo "Usage: $0 {start|stop|restart|reload|force-reload}" >&2 + exit 1 + ;; +esac + +exit 0 From bf4418bced5ba159b98f4bf15cd2dca2ad8a7690 Mon Sep 17 00:00:00 2001 From: achbed Date: Sun, 30 Nov 2014 18:26:06 -0600 Subject: [PATCH 32/82] Added new QueueLogger class (logging from a queue so we can do thread-safe logging) Moved core and station to new common log functions (_info and _err) using new QueueLogger Core now attempts to restart stations that stop Core now detects new folders on the fly instead of only at start time (stationfolder option) Station.py has a lot of changes to utilize the common blocking queue only when needed rather than at every step (may increase performance with many stations) TODO: Detect proper vs unexpected shutdown of stations and restart only unexpected ones Make new folder detection for stationfolder an option --- deefuzzer/core.py | 147 ++++++++++++------ deefuzzer/station.py | 306 +++++++++++++++++++++----------------- deefuzzer/tools/logger.py | 29 ++++ 3 files changed, 299 insertions(+), 183 deletions(-) diff --git a/deefuzzer/core.py b/deefuzzer/core.py index 9e09987..1f19437 100644 --- a/deefuzzer/core.py +++ b/deefuzzer/core.py @@ -54,8 +54,10 @@ class DeeFuzzer(Thread): logger = None m3u = None rss = None - station = [] - stations = [] + station_settings = [] + station_instances = {} + watchfolder = {} + logqueue = Queue.Queue() def __init__(self, conf_file): Thread.__init__(self) @@ -70,7 +72,8 @@ def __init__(self, conf_file): log_dir = os.sep.join(log_file.split(os.sep)[:-1]) if not os.path.exists(log_dir) and log_dir: os.makedirs(log_dir) - self.logger = Logger(log_file) + self.logger = QueueLogger(log_file, self.logqueue) + self.logger.start() for key in self.conf['deefuzzer'].keys(): if key == 'm3u': @@ -91,31 +94,48 @@ def __init__(self, conf_file): elif key == 'stationfolder': # Create stations automagically from a folder structure if isinstance(self.conf['deefuzzer'][key], dict): - self.create_stations_fromfolder(self.conf['deefuzzer'][key]) + self.watchfolder = self.conf['deefuzzer'][key] else: setattr(self, key, self.conf['deefuzzer'][key]) # Set the deefuzzer logger - self.logger.write_info('Starting DeeFuzzer') - self.logger.write_info('Using libshout version %s' % shout.version()) - self.logger.write_info('Number of stations : ' + str(len(self.station))) + self._info('Starting DeeFuzzer') + self._info('Using libshout version %s' % shout.version()) + self._info('Number of stations : ' + str(len(self.station_settings))) + def _log(self, level, msg): + try: + obj = {} + obj['msg'] = 'Core: ' + str(msg) + obj['level'] = level + self.logqueue.put(obj) + except: + pass + + def _info(self, msg): + self._log('info', msg) + + def _err(self, msg): + self._log('err', msg) + def set_m3u_playlist(self): m3u_dir = os.sep.join(self.m3u.split(os.sep)[:-1]) if not os.path.exists(m3u_dir) and m3u_dir: os.makedirs(m3u_dir) m3u = open(self.m3u, 'w') m3u.write('#EXTM3U\n') - for s in self.stations: + for k in self.station_instances.keys(): + s = self.station_instances[k] m3u.write('#EXTINF:%s,%s - %s\n' % ('-1',s.short_name, s.channel.name)) m3u.write('http://' + s.channel.host + ':' + str(s.channel.port) + s.channel.mount + '\n') m3u.close() - self.logger.write_info('Writing M3U file to : ' + self.m3u) + self._info('Writing M3U file to : ' + self.m3u) - def create_stations_fromfolder(self, options): + def create_stations_fromfolder(self): """Scan a folder for subfolders containing media, and make stations from them all.""" + options = self.watchfolder if not 'folder' in options.keys(): # We have no folder specified. Bail. return @@ -125,7 +145,8 @@ def create_stations_fromfolder(self, options): # The specified path is not a folder. Bail. return - self.logger.write_info('Scanning folder ' + folder + ' for stations') + # This makes the log file a lot more verbose. Commented out since we report on new stations anyway. + # self._info('Scanning folder ' + folder + ' for stations') files = os.listdir(folder) for file in files: @@ -134,12 +155,28 @@ def create_stations_fromfolder(self, options): if folder_contains_music(filepath): self.create_station(filepath, options) + def station_exists(self, name): + try: + for s in self.station_settings: + if not 'infos' in s.keys(): + continue + if not 'short_name' in s['infos'].keys(): + continue + if s['infos']['short_name'] == name: + return True + return False + except: + pass + return True + def create_station(self, folder, options): """Create a station definition for a folder given the specified options.""" - self.logger.write_info('Creating station for folder ' + folder) s = {} path, name = os.path.split(folder) + if self.station_exists(name): + return + self._info('Creating station for folder ' + folder) d = dict(path=folder,name=name) for i in options.keys(): if not 'folder' in i: @@ -167,7 +204,7 @@ def load_stations_fromconfig(self, folder): # Whatever we have, it's not either a file or folder. Bail. return - self.logger.write_info('Loading station config files in ' + folder) + self._info('Loading station config files in ' + folder) files = os.listdir(folder) for file in files: filepath = os.path.join(folder, file) @@ -177,7 +214,7 @@ def load_stations_fromconfig(self, folder): def load_station_config(self, file): """Load station configuration(s) from a config file.""" - self.logger.write_info('Loading station config file ' + file) + self._info('Loading station config file ' + file) stationdef = get_conf_dict(file) if isinstance(stationdef, dict): if 'station' in stationdef.keys(): @@ -192,48 +229,66 @@ def add_station(self, this_station): try: # We should probably test to see if we're putting the same station in multiple times # Same in this case probably means the same media folder, server, and mountpoint - self.station.append(this_station) + self.station_settings.append(this_station) except Exception: return def run(self): q = Queue.Queue(1) - - ns = len(self.station) - for i in range(0, ns): - try: - station = self.station[i] - - # Apply station defaults if they exist - if 'stationdefaults' in self.conf['deefuzzer']: - if isinstance(self.conf['deefuzzer']['stationdefaults'], dict): - station = merge_defaults(station, self.conf['deefuzzer']['stationdefaults']) - self.stations.append(Station(station, q, self.logger, self.m3u)) - except Exception: - name = str(i) - if 'info' in station.keys(): - if 'short_name' in station['infos']: - name = station['infos']['short_name'] - self.logger.write_error('Error starting station ' + name) - continue - - if self.m3u: - self.set_m3u_playlist() - + ns = -1 p = Producer(q) p.start() - - ns = len(self.stations) - # Start the Stations - for i in range(0, ns): - try: - self.stations[i].start() - except Exception: - continue + started = False + # Keep the Stations running + while True: + self.create_stations_fromfolder() + ns_new = len(self.station_settings) + if(ns_new > ns): + for i in range(ns+1, ns_new): + try: + station = self.station_settings[i] + + # Apply station defaults if they exist + if 'stationdefaults' in self.conf['deefuzzer']: + if isinstance(self.conf['deefuzzer']['stationdefaults'], dict): + station = merge_defaults(station, self.conf['deefuzzer']['stationdefaults']) + + name = 'Station ' + str(i) + if 'info' in station.keys(): + if 'short_name' in station['infos']: + name = station['infos']['short_name'] + y = 1 + while name in self.station_instances.keys(): + y = y + 1 + name = station['infos']['short_name'] + " " + str(y) + + self.station_instances[name] = Station(station, q, self.logqueue, self.m3u) + except Exception: + self._err('Error starting station ' + name) + continue + ns = ns_new + + if self.m3u: + self.set_m3u_playlist() + + for i in self.station_instances.keys(): + try: + if not self.station_instances[i].isAlive(): + self.station_instances[i].start() + msg = 'Started ' + if started: + msg = 'Restarted ' + self._info(msg + 'station ' + i) + except: + pass + + started = False + time.sleep(5) + # end main loop class Producer(Thread): - """a DeeFuzzer Producer master thread""" + """a DeeFuzzer Producer master thread. Used for locking/blocking""" def __init__(self, q): Thread.__init__(self) diff --git a/deefuzzer/station.py b/deefuzzer/station.py index 0397383..a433d68 100644 --- a/deefuzzer/station.py +++ b/deefuzzer/station.py @@ -80,12 +80,13 @@ class Station(Thread): feeds_playlist = 1 feeds_showfilepath = 0 feeds_showfilename = 0 + short_name = '' - def __init__(self, station, q, logger, m3u): + def __init__(self, station, q, logqueue, m3u): Thread.__init__(self) self.station = station self.q = q - self.logger = logger + self.logqueue = logqueue self.m3u = m3u if 'station_dir' in self.station: @@ -185,8 +186,7 @@ def __init__(self, station, q, logger, m3u): self.feeds_playlist_file = self.base_name + '_playlist' # Logging - self._info('Opening ' + self.short_name + ' - ' + self.channel.name + \ - ' (' + str(self.lp) + ' tracks)...') + self._info('Opening ' + self.short_name + ' - ' + self.channel.name) self.metadata_relative_dir = 'metadata' self.metadata_url = self.channel.url + '/rss/' + self.metadata_relative_dir @@ -253,12 +253,21 @@ def __init__(self, station, q, logger, m3u): if self.record_mode: self.record_callback('/record', [1]) + def _log(self, level, msg): + try: + obj = {} + obj['msg'] = 'Station ' + str(self.channel_url) + ': ' + str(msg) + obj['level'] = str(level) + self.logqueue.put(obj) + except: + pass + def _info(self, msg): - self.logger.write_info('Station ' + self.channel_url + ': ' + str(msg)) + self._log('info', msg) def _err(self, msg): - self.logger.write_error('Station ' + self.channel_url + ': ' + str(msg)) - + self._log('err', msg) + def run_callback(self, path, value): value = value[0] self.run_mode = value @@ -359,10 +368,18 @@ def get_playlist(self): file_list.append(root + os.sep + file) file_list.sort() else: - f = open(self.m3u_playlist_file, 'r') - for path in f.readlines(): - if '#' != path[0]: - file_list.append(path[:-1]) + self.q.get(1) + try: + f = open(self.m3u_playlist_file, 'r') + try: + for path in f.readlines(): + if '#' != path[0]: + file_list.append(path[:-1]) + except: + f.close() + except: + pass + self.q.task_done() return file_list def get_jingles(self): @@ -416,7 +433,7 @@ def get_next_media(self): if self.shuffle_mode: random.shuffle(self.playlist) - self._info('generating new playlist (' + str(self.lp) + ' tracks)') + self._info('Generating new playlist (' + str(self.lp) + ' tracks)') if self.feeds_playlist: self.update_feeds(self.media_to_objs(self.playlist), self.feeds_playlist_file, '(playlist)') @@ -459,31 +476,36 @@ def get_next_media(self): else: mess = 'No media in media_dir !' self._err(mess) - sys.exit(mess) + self.run_mode = 0 def media_to_objs(self, media_list): media_objs = [] for media in media_list: file_name, file_title, file_ext = get_file_info(media) - if file_ext.lower() == 'mp3' or mimetypes.guess_type(media)[0] == 'audio/mpeg': - try: - file_meta = Mp3(media) - except: - continue - elif file_ext.lower() == 'ogg' or mimetypes.guess_type(media)[0] == 'audio/ogg': - try: - file_meta = Ogg(media) - except: - continue - elif file_ext.lower() == 'webm' or mimetypes.guess_type(media)[0] == 'video/webm': - try: - file_meta = WebM(media) - except: - continue - if self.feeds_showfilename: - file_meta.metadata['filename'] = file_name.decode( "utf-8" ) #decode needed for some weird filenames - if self.feeds_showfilepath: - file_meta.metadata['filepath'] = media.decode( "utf-8" ) #decode needed for some weird filenames + self.q.get(1) + try: + if file_ext.lower() == 'mp3' or mimetypes.guess_type(media)[0] == 'audio/mpeg': + try: + file_meta = Mp3(media) + except: + continue + elif file_ext.lower() == 'ogg' or mimetypes.guess_type(media)[0] == 'audio/ogg': + try: + file_meta = Ogg(media) + except: + continue + elif file_ext.lower() == 'webm' or mimetypes.guess_type(media)[0] == 'video/webm': + try: + file_meta = WebM(media) + except: + continue + if self.feeds_showfilename: + file_meta.metadata['filename'] = file_name.decode( "utf-8" ) #decode needed for some weird filenames + if self.feeds_showfilepath: + file_meta.metadata['filepath'] = media.decode( "utf-8" ) #decode needed for some weird filenames + except: + pass + self.q.task_done() media_objs.append(file_meta) return media_objs @@ -554,21 +576,25 @@ def update_feeds(self, media_list, rss_file, sub_title): ) json_data.append(json_item) - rss = RSS2(title = channel_subtitle, - link = self.channel.url, - description = self.channel.description.decode('utf-8'), - lastBuildDate = date_now, + rss = RSS2(title = channel_subtitle, \ + link = self.channel.url, \ + description = self.channel.description.decode('utf-8'), \ + lastBuildDate = date_now, \ items = rss_item_list,) - - if self.feeds_rss: - f = open(rss_file + '.xml', 'w') - rss.write_xml(f, 'utf-8') - f.close() - - if self.feeds_json: - f = open(rss_file + '.json', 'w') - f.write(json.dumps(json_data, separators=(',',':'))) - f.close() + self.q.get(1) + try: + if self.feeds_rss: + f = open(rss_file + '.xml', 'w') + rss.write_xml(f, 'utf-8') + f.close() + + if self.feeds_json: + f = open(rss_file + '.json', 'w') + f.write(json.dumps(json_data, separators=(',',':'))) + f.close() + except: + pass + self.q.task_done() def update_twitter(self, message): try: @@ -619,10 +645,15 @@ def set_read_mode(self): % (self.id, self.current_media_obj[0].file_name)) self.player.set_media(self.media) - if self.player_mode: - self.stream = self.player.file_read_slow() - else: - self.stream = self.player.file_read_fast() + self.q.get(1) + try: + if self.player_mode: + self.stream = self.player.file_read_slow() + else: + self.stream = self.player.file_read_fast() + except: + pass + self.q.task_done() def set_webm_read_mode(self): self.channel.set_callback(FileReader(self.media).read_callback) @@ -645,53 +676,48 @@ def ping_server(self): while not self.server_ping: try: - self.q.get(1) server = urllib.urlopen(self.server_url) self.server_ping = True - text = 'channel available' - self._info(text) - self.q.task_done() + self._info('Channel available.') except: time.sleep(1) if log: - text = 'could not connect the channel' - self._err(text) + self._err('Could not connect the channel. Waiting for channel to become available.') log = False - self.q.task_done() def icecastloop_nextmedia(self): - self.q.get(1) - self.next_media = 0 - self.media = self.get_next_media() - self.counter += 1 - if self.relay_mode: - self.set_relay_mode() - elif os.path.exists(self.media) and not os.sep+'.' in self.media: - if self.lp == 0: - self._err('has no media to stream !') - return False - self.set_read_mode() - self.q.task_done() - - return True + try: + self.next_media = 0 + self.media = self.get_next_media() + self.counter += 1 + if self.relay_mode: + self.set_relay_mode() + elif os.path.exists(self.media) and not os.sep+'.' in self.media: + if self.lp == 0: + self._err('has no media to stream !') + return False + self.set_read_mode() + + return True + except Exception, e: + self_err('icecastloop_nextmedia: Error: ' + str(e)) + return False def icecastloop_metadata(self): - self.q.get(1) - if (not (self.jingles_mode and (self.counter % 2)) or \ - self.relay_mode) and self.twitter_mode: - self.update_twitter_current() - self.channel.set_metadata({'song': self.song, 'charset': 'utf-8'}) - self.q.task_done() - return True + try: + if (not (self.jingles_mode and (self.counter % 2)) or \ + self.relay_mode) and self.twitter_mode: + self.update_twitter_current() + self.channel.set_metadata({'song': self.song, 'charset': 'utf-8'}) + return True + except Exception, e: + self_err('icecastloop_metadata: Error: ' + str(e)) + return False - def run(self): - self.q.get(1) self.ping_server() - self.q.task_done() if self.type == 'stream-m': - self.q.get(1) if self.relay_mode: self.set_relay_mode() else: @@ -699,76 +725,82 @@ def run(self): self.set_webm_read_mode() self.channel_open() self.channel.start() - self.q.task_done() if self.type == 'icecast': - self.q.get(1) self.channel_open() self._info('channel connected') - self.q.task_done() while self.run_mode: - try: - if not self.icecastloop_nextmedia(): - break - if not self.icecastloop_metadata(): + if not self.icecastloop_nextmedia(): + self._info('Something wrong happened in icecastloop_nextmedia. Ending.') + break + + self.icecastloop_metadata() + + # TEST MODE: Jump thru only the first chunk of each file + # first = True + for self.chunk in self.stream: + # if first: + # first = False + # else: + # break + + if self.next_media or not self.run_mode: break - - for self.chunk in self.stream: + + if self.record_mode: try: - if self.next_media or not self.run_mode: - break + # Record the chunk + self.recorder.write(self.chunk) + except: + self._err('could not write the buffer to the file') + continue + try: + # Send the chunk to the stream + self.channel.send(self.chunk) + self.channel.sync() + except: + self._err('could not send the buffer') + + try: + self.channel.close() + self._info('channel closed') + except: + self._err('could not close the channel') + self.q.task_done() + continue + + try: + self.ping_server() + self.channel_open() + self.channel.set_metadata({'song': self.song, 'charset': 'utf8',}) + self._info('channel restarted') + except: + self._err('could not restart the channel') if self.record_mode: - try: - self.q.get(1) - self.recorder.write(self.chunk) - self.q.task_done() - except: - self._err('could not write the buffer to the file') - self.q.task_done() - continue - + self.recorder.close() + return + + try: + self.channel.send(self.chunk) + self.channel.sync() + except: + self._err('could not send data after restarting the channel') try: - self.q.get(1) - self.channel.send(self.chunk) - self.channel.sync() - self.q.task_done() + self.channel.close() except: - self._err('could not send the buffer') - self.q.task_done() + self._err('could not close the channel') - try: - self.q.get(1) - self.channel.close() - self._info('channel closed') - self.q.task_done() - except: - self._err('could not close the channel') - self.q.task_done() - continue - - try: - self.ping_server() - self.q.get(1) - self.channel_open() - self.channel.set_metadata({'song': self.song, 'charset': 'utf8',}) - self._info('channel restarted') - self.q.task_done() - except: - self._err('could not restart the channel') - self.q.task_done() - continue - continue - - except: # send chunk loop exception - continue - # send chunk loop end - - except: # while run_mode exception - continue + if self.record_mode: + self.recorder.close() + return + + # send chunk loop end # while run_mode loop end + self._info("Play mode ended. Stopping stream.") + if self.record_mode: self.recorder.close() diff --git a/deefuzzer/tools/logger.py b/deefuzzer/tools/logger.py index 041f04c..5bb6570 100644 --- a/deefuzzer/tools/logger.py +++ b/deefuzzer/tools/logger.py @@ -2,6 +2,7 @@ # -*- coding: utf-8 -*- import logging +from threading import Thread class Logger: @@ -21,3 +22,31 @@ def write_info(self, message): def write_error(self, message): self.logger.error(message) +class QueueLogger(Thread): + """A queue-based logging object""" + + def __init__(self, file, q): + Thread.__init__(self) + self.logger = Logger(file) + self.q = q + + def run(self): + while True: + try: + msg = self.q.get(1) + if not isinstance(msg, dict): + self.logger.write_error(str(msg)) + else: + if not 'msg' in msg.keys(): + continue + + if 'level' in msg.keys(): + if msg['level'] == 'info': + self.logger.write_info(msg['msg']) + else: + self.logger.write_error(msg['msg']) + else: + self.logger.write_error(msg['msg']) + except: + pass + \ No newline at end of file From 5fd4bd9fbb76abf3958e16e91a4d50a976012177 Mon Sep 17 00:00:00 2001 From: achbed Date: Mon, 1 Dec 2014 00:37:45 -0600 Subject: [PATCH 33/82] Revised crashed station detection Added main loop in station to allow remote commands to restart playback after stopping Added livecreation option to enable.disable watchfolder mode Added fix for newly created stations not starting Cleaned up a missed queue unlock instance Signed-off-by: achbed --- deefuzzer/core.py | 34 ++++++--- deefuzzer/station.py | 153 ++++++++++++++++++++------------------ example/deefuzzer_doc.xml | 3 + 3 files changed, 108 insertions(+), 82 deletions(-) diff --git a/deefuzzer/core.py b/deefuzzer/core.py index 1f19437..f34cecc 100644 --- a/deefuzzer/core.py +++ b/deefuzzer/core.py @@ -58,6 +58,7 @@ class DeeFuzzer(Thread): station_instances = {} watchfolder = {} logqueue = Queue.Queue() + mainLoop = False def __init__(self, conf_file): Thread.__init__(self) @@ -140,6 +141,15 @@ def create_stations_fromfolder(self): # We have no folder specified. Bail. return + if self.mainLoop: + if not 'livecreation' in options.keys(): + # We have no folder specified. Bail. + return + + if int(options['livecreation']) == 0: + # Livecreation not specified. Bail. + return + folder = str(options['folder']) if not os.path.isdir(folder): # The specified path is not a folder. Bail. @@ -235,18 +245,20 @@ def add_station(self, this_station): def run(self): q = Queue.Queue(1) - ns = -1 + ns = 0 p = Producer(q) p.start() - started = False # Keep the Stations running while True: self.create_stations_fromfolder() ns_new = len(self.station_settings) if(ns_new > ns): - for i in range(ns+1, ns_new): + self._info('Loading new stations') + for i in range(0, ns_new): try: station = self.station_settings[i] + if 'station_name' in station.keys(): + continue # Apply station defaults if they exist if 'stationdefaults' in self.conf['deefuzzer']: @@ -262,7 +274,14 @@ def run(self): y = y + 1 name = station['infos']['short_name'] + " " + str(y) - self.station_instances[name] = Station(station, q, self.logqueue, self.m3u) + self.station_settings[i]['station_name'] = name + new_station = Station(station, q, self.logqueue, self.m3u) + if new_station.valid: + self.station_instances[name] = new_station + self.station_instances[name].start() + self._info('Started station ' + name) + else: + self._err('Error validating station ' + name) except Exception: self._err('Error starting station ' + name) continue @@ -275,14 +294,11 @@ def run(self): try: if not self.station_instances[i].isAlive(): self.station_instances[i].start() - msg = 'Started ' - if started: - msg = 'Restarted ' - self._info(msg + 'station ' + i) + self._info('Restarted crashed station ' + i) except: pass - started = False + self.mainLoop = True time.sleep(5) # end main loop diff --git a/deefuzzer/station.py b/deefuzzer/station.py index a433d68..691c4f6 100644 --- a/deefuzzer/station.py +++ b/deefuzzer/station.py @@ -59,6 +59,7 @@ class Station(Thread): """a DeeFuzzer shouting station thread""" id = 999999 + valid = False counter = 0 delay = 0 start_time = time.time() @@ -81,6 +82,7 @@ class Station(Thread): feeds_showfilepath = 0 feeds_showfilename = 0 short_name = '' + channelIsOpen = False def __init__(self, station, q, logqueue, m3u): Thread.__init__(self) @@ -131,7 +133,8 @@ def __init__(self, station, q, logqueue, m3u): if self.appendtype: self.channel.mount = self.channel.mount + '.' + self.media_format else: - sys.exit('Not a compatible server type. Choose "stream-m" or "icecast".') + self._err('Not a compatible server type. Choose "stream-m" or "icecast".') + return self.channel.url = self.station['infos']['url'] self.channel.name = self.station['infos']['name'] @@ -253,6 +256,8 @@ def __init__(self, station, q, logqueue, m3u): if self.record_mode: self.record_callback('/record', [1]) + self.valid = True + def _log(self, level, msg): try: obj = {} @@ -668,8 +673,24 @@ def update_twitter_current(self): self.update_twitter(message) def channel_open(self): - self.channel.open() - self.channel_delay = self.channel.delay() + if not self.channelIsOpen: + try: + self.channel.open() + self.channel_delay = self.channel.delay() + self._info('channel connected') + self.channelIsOpen = True + return True + except: + self.err('channel could not be opened') + return False + + def channel_close(self): + self.channelIsOpen = False + try: + self.channel.close() + self._info('channel closed') + except: + self._err('channel could not be closed') def ping_server(self): log = True @@ -723,86 +744,72 @@ def run(self): else: self.media = self.get_next_media() self.set_webm_read_mode() - self.channel_open() + if not self.channel_open(): + return self.channel.start() if self.type == 'icecast': - self.channel_open() - self._info('channel connected') - - while self.run_mode: - if not self.icecastloop_nextmedia(): - self._info('Something wrong happened in icecastloop_nextmedia. Ending.') - break - - self.icecastloop_metadata() - - # TEST MODE: Jump thru only the first chunk of each file - # first = True - for self.chunk in self.stream: - # if first: - # first = False - # else: - # break - - if self.next_media or not self.run_mode: - break - - if self.record_mode: - try: - # Record the chunk - self.recorder.write(self.chunk) - except: - self._err('could not write the buffer to the file') - continue - - try: - # Send the chunk to the stream - self.channel.send(self.chunk) - self.channel.sync() - except: - self._err('could not send the buffer') + while True: # Do this so that the handlers will still restart the stream + while self.run_mode: + if not self.channel_open(): + return + if not self.icecastloop_nextmedia(): + self._info('Something wrong happened in icecastloop_nextmedia. Ending.') + return + + self.icecastloop_metadata() + + # TEST MODE: Jump thru only the first chunk of each file + # first = True + for self.chunk in self.stream: + # if first: + # first = False + # else: + # break + + if self.next_media or not self.run_mode: + break + + if self.record_mode: + try: + # Record the chunk + self.recorder.write(self.chunk) + except: + self._err('could not write the buffer to the file') + try: - self.channel.close() - self._info('channel closed') - except: - self._err('could not close the channel') - self.q.task_done() - continue - - try: - self.ping_server() - self.channel_open() - self.channel.set_metadata({'song': self.song, 'charset': 'utf8',}) - self._info('channel restarted') - except: - self._err('could not restart the channel') - if self.record_mode: - self.recorder.close() - return - - try: + # Send the chunk to the stream self.channel.send(self.chunk) self.channel.sync() except: - self._err('could not send data after restarting the channel') + self._err('could not send the buffer') + self.channel_close() + if not self.channel_open(): + self._err('could not restart the channel') + if self.record_mode: + self.recorder.close() + return try: - self.channel.close() + self.channel.set_metadata({'song': self.song, 'charset': 'utf8',}) + self._info('channel restarted') + self.channel.send(self.chunk) + self.channel.sync() except: - self._err('could not close the channel') + self._err('could not send data after restarting the channel') + self.channel_close() + if self.record_mode: + self.recorder.close() + return - if self.record_mode: - self.recorder.close() - return - - # send chunk loop end - # while run_mode loop end - - self._info("Play mode ended. Stopping stream.") - - if self.record_mode: - self.recorder.close() + # send chunk loop end + # while run_mode loop end + + self._info("Play mode ended. Stopping stream.") + + if self.record_mode: + self.recorder.close() - self.channel.close() + self.channel_close() + time.sleep(1) diff --git a/example/deefuzzer_doc.xml b/example/deefuzzer_doc.xml index 138a6a9..6a3dba3 100644 --- a/example/deefuzzer_doc.xml +++ b/example/deefuzzer_doc.xml @@ -151,6 +151,9 @@ /path/to/media + + 1 /path/to/station.log /path/to/station.m3u + + 0 0 + + 0 0 - + 1 @@ -60,13 +65,13 @@ /path/to/jingles 0 - + 1 96 - + /path/to/mp3/ mp3 @@ -74,9 +79,9 @@ /path/to/m3u_file 4 - + 48000 - + 0 2 @@ -150,7 +155,7 @@ - + From 0077d37ab3e5815f7f02e52fe7a2e060b734b4f9 Mon Sep 17 00:00:00 2001 From: achbed Date: Fri, 16 Jan 2015 16:48:37 -0600 Subject: [PATCH 44/82] Fixed logic flaw with retry counts (negative maxretry value would never retry instead of always retry) Signed-off-by: achbed --- deefuzzer/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deefuzzer/core.py b/deefuzzer/core.py index 60648a2..0e0fcab 100644 --- a/deefuzzer/core.py +++ b/deefuzzer/core.py @@ -283,7 +283,7 @@ def run(self): self.station_settings[i]['retries'] = 0 continue - if self.maxretry < 0 or self.station_settings[i]['retries'] <= self.maxretry: + if self.maxretry >= 0 and self.station_settings[i]['retries'] <= self.maxretry: # Station passed the max retries count is will not be reloaded continue From dc35024d541913e648ed660283a989258a64339f Mon Sep 17 00:00:00 2001 From: achbed Date: Mon, 19 Jan 2015 16:58:29 -0600 Subject: [PATCH 45/82] Additional logging when a station was found stopped and will never be restarted Better description in logging station initialization errors Internal station name and status filenames will no longer be regenerated on restart Signed-off-by: achbed --- deefuzzer/core.py | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/deefuzzer/core.py b/deefuzzer/core.py index 0e0fcab..817bde4 100644 --- a/deefuzzer/core.py +++ b/deefuzzer/core.py @@ -285,6 +285,9 @@ def run(self): if self.maxretry >= 0 and self.station_settings[i]['retries'] <= self.maxretry: # Station passed the max retries count is will not be reloaded + if not 'station_stop_logged' in self.station_settings[i].keys(): + self._err('Station ' + name + ' is stopped and will not be restarted.') + self.station_settings[i]['station_stop_logged'] = True continue self.station_settings[i]['retries'] = self.station_settings[i]['retries'] + 1 @@ -302,18 +305,17 @@ def run(self): if name == '': name = 'Station ' + str(i) - - if 'info' in self.station_settings[i].keys(): - if 'short_name' in self.station_settings[i]['infos']: - name = self.station_settings[i]['infos']['short_name'] - y = 1 - while name in self.station_instances.keys(): - y = y + 1 - name = self.station_settings[i]['infos']['short_name'] + " " + str(y) - - self.station_settings[i]['station_name'] = name - namehash = hashlib.md5(name).hexdigest() - self.station_settings[i]['station_statusfile'] = os.sep.join([self.log_dir, namehash]) + if 'info' in self.station_settings[i].keys(): + if 'short_name' in self.station_settings[i]['infos']: + name = self.station_settings[i]['infos']['short_name'] + y = 1 + while name in self.station_instances.keys(): + y = y + 1 + name = self.station_settings[i]['infos']['short_name'] + " " + str(y) + + self.station_settings[i]['station_name'] = name + namehash = hashlib.md5(name).hexdigest() + self.station_settings[i]['station_statusfile'] = os.sep.join([self.log_dir, namehash]) new_station = Station(self.station_settings[i], q, self.logqueue, self.m3u) if new_station.valid: @@ -323,7 +325,7 @@ def run(self): else: self._err('Error validating station ' + name) except Exception: - self._err('Error creating station ' + name) + self._err('Error initializing station ' + name) if not ignoreErrors: raise continue From 871bf9da94b821d2662e61084acb93175c6ca9d4 Mon Sep 17 00:00:00 2001 From: achbed Date: Mon, 19 Jan 2015 18:16:59 -0600 Subject: [PATCH 46/82] Changes based on ChoiZ's mode_playing branch Station: media.dir now overrides media.m3u if both are specified and non-blank Station: Noted media.m3u and media.dir options as depreciated (uses them if present for smooth upgrade path) Station: Combine media.m3u and media.dir options into one new media.source option Station: media.source is now checked to see if it's a file or folder at load time, and playlist or folder list mode is selected automatically Core: Updated to reflect new source setting when using station autocreation Signed-off-by: achbed --- deefuzzer/core.py | 2 +- deefuzzer/station.py | 73 +++++++++++++++++++++++------------ example/deefuzzer.json | 3 +- example/deefuzzer.xml | 3 +- example/deefuzzer.yaml | 3 +- example/deefuzzer_doc.xml | 16 +++++--- example/stationconfig_doc.xml | 3 +- 7 files changed, 64 insertions(+), 39 deletions(-) diff --git a/deefuzzer/core.py b/deefuzzer/core.py index 817bde4..e394c92 100644 --- a/deefuzzer/core.py +++ b/deefuzzer/core.py @@ -204,7 +204,7 @@ def create_station(self, folder, options): s[i] = replace_all(options[i], d) if not 'media' in s.keys(): s['media'] = {} - s['media']['dir'] = folder + s['media']['source'] = folder self.add_station(s) def load_stations_fromconfig(self, folder): diff --git a/deefuzzer/station.py b/deefuzzer/station.py index c6c92b9..566b8e5 100644 --- a/deefuzzer/station.py +++ b/deefuzzer/station.py @@ -65,6 +65,7 @@ class Station(Thread): start_time = time.time() server_ping = False playlist = [] + media_source = None lp = 1 player_mode = 1 osc_control_mode = 0 @@ -94,7 +95,7 @@ def __init__(self, station, q, logqueue, m3u): self.logqueue = logqueue self.m3u = m3u - if 'station_statusfile' in station: + if 'station_statusfile' in self.station: self.statusfile = station['station_statusfile'] try: if os.path.exists(self.statusfile): @@ -108,16 +109,24 @@ def __init__(self, station, q, logqueue, m3u): self.station_dir = self.station['station_dir'] # Media - self.media_dir = self.station['media']['dir'] + if 'm3u' in self.station['media'].keys(): + if not self.station['media']['m3u'].strip() == '': + self.media_source = self.station['media']['m3u'] + + if 'dir' in self.station['media'].keys(): + if not self.station['media']['dir'].strip() == '': + self.media_source = self.station['media']['dir'] + + if 'source' in self.station['media'].keys(): + if not self.station['media']['source'].strip() == '': + self.media_source = self.station['media']['source'] + self.media_format = self.station['media']['format'] self.shuffle_mode = int(self.station['media']['shuffle']) self.bitrate = int(self.station['media']['bitrate']) self.ogg_quality = int(self.station['media']['ogg_quality']) self.samplerate = int(self.station['media']['samplerate']) self.voices = int(self.station['media']['voices']) - self.m3u_playlist_file = [] - if 'm3u' in self.station['media'].keys(): - self.m3u_playlist_file = self.station['media']['m3u'] # Server if 'mountpoint' in self.station['server'].keys(): @@ -326,6 +335,9 @@ def twitter_callback(self, path, value): self.twitter = Twitter(self.twitter_key, self.twitter_secret) self.twitter_mode = value message = "received OSC message '%s' with arguments '%d'" % (path, value) + + # IMPROVEMENT: The URL paths should be configurable because they're + # server-implementation specific self.m3u_url = self.channel.url + '/m3u/' + self.m3u.split(os.sep)[-1] self.feeds_url = self.channel.url + '/rss/' + self.feeds_playlist_file.split(os.sep)[-1] self._info(message) @@ -382,27 +394,38 @@ def player_callback(self, path, value): def get_playlist(self): file_list = [] - if not self.m3u_playlist_file: - for root, dirs, files in os.walk(self.media_dir): - for file in files: - s = file.split('.') - ext = s[len(s)-1] - if ext.lower() == self.channel.format and not os.sep+'.' in file: - file_list.append(root + os.sep + file) - file_list.sort() - else: - self.q.get(1) - try: - f = open(self.m3u_playlist_file, 'r') + + try: + if os.path.isdir(self.media_source): + self.q.get(1) try: - for path in f.readlines(): - if '#' != path[0]: - file_list.append(path[:-1]) + for root, dirs, files in os.walk(self.media_source): + for file in files: + s = file.split('.') + ext = s[len(s)-1] + if ext.lower() == self.channel.format and not os.sep+'.' in file: + file_list.append(root + os.sep + file) + file_list.sort() except: - f.close() - except: - pass - self.q.task_done() + pass + self.q.task_done() + + if os.path.isfile(self.media_source): + self.q.get(1) + try: + f = open(self.media_source, 'r') + try: + for path in f.readlines(): + if '#' != path[0]: + file_list.append(path[:-1]) + except: + f.close() + except: + pass + self.q.task_done() + except: + pass + return file_list def get_jingles(self): @@ -511,7 +534,7 @@ def get_next_media(self): self.q.task_done() return media else: - mess = 'No media in media_dir!' + mess = 'No media in source!' self._err(mess) self.run_mode = 0 diff --git a/example/deefuzzer.json b/example/deefuzzer.json index 7f53f86..24a71bb 100644 --- a/example/deefuzzer.json +++ b/example/deefuzzer.json @@ -43,9 +43,8 @@ }, "media": { "bitrate": 96, - "dir": "/path/to/mp3/", + "source": "/path/to/mp3/or/m3u", "format": "mp3", - "m3u": "/path/to/m3u_file", "ogg_quality": 4, "samplerate": 48000, "shuffle": 0, diff --git a/example/deefuzzer.xml b/example/deefuzzer.xml index 9cadbb1..6c16692 100644 --- a/example/deefuzzer.xml +++ b/example/deefuzzer.xml @@ -31,9 +31,8 @@ 96 - /path/to/mp3/ + /path/to/mp3/or/m3u mp3 - /path/to/m3u_file 4 48000 0 diff --git a/example/deefuzzer.yaml b/example/deefuzzer.yaml index 366cb7f..ffbe47b 100644 --- a/example/deefuzzer.yaml +++ b/example/deefuzzer.yaml @@ -25,9 +25,8 @@ deefuzzer: shuffle: 1} media: {bitrate: 96, - dir: /path/to/mp3/, + source: /path/to/mp3/or/m3u, format: mp3, - m3u: /path/to/m3u_file, ogg_quality: 4, samplerate: 48000, shuffle: 0, diff --git a/example/deefuzzer_doc.xml b/example/deefuzzer_doc.xml index b51348a..93dcc3e 100644 --- a/example/deefuzzer_doc.xml +++ b/example/deefuzzer_doc.xml @@ -63,7 +63,8 @@ /path/to/jingles - + 0 1 @@ -71,12 +72,17 @@ 96 - - /path/to/mp3/ + + /path/to/m3u_file + + /path/to/mp3_folder + + /path/to/m3u_file_or_folder mp3 - - /path/to/m3u_file 4 diff --git a/example/stationconfig_doc.xml b/example/stationconfig_doc.xml index 740b3c0..17cec3b 100644 --- a/example/stationconfig_doc.xml +++ b/example/stationconfig_doc.xml @@ -16,9 +16,8 @@ 96 - /path/to/mp3/ + /path/to/mp3/or/m3u mp3 - /path/to/m3u_file 4 48000 0 From 5930f22b6ed0bbc465185abdae9fe195a30bb8d8 Mon Sep 17 00:00:00 2001 From: achbed Date: Sat, 24 Jan 2015 01:54:41 -0600 Subject: [PATCH 47/82] Updates for relative paths Added base_dir parameter to station definition (issue #10) Added default station short_name to auto-created stations Removed previously-mandatory tags from example XML config stationfolder tag Signed-off-by: achbed --- deefuzzer/core.py | 6 ++++++ deefuzzer/station.py | 25 ++++++++++++++----------- example/deefuzzer_doc.xml | 19 +++++++++++++------ 3 files changed, 33 insertions(+), 17 deletions(-) diff --git a/deefuzzer/core.py b/deefuzzer/core.py index e394c92..ab95474 100644 --- a/deefuzzer/core.py +++ b/deefuzzer/core.py @@ -169,6 +169,11 @@ def create_stations_fromfolder(self): # This makes the log file a lot more verbose. Commented out since we report on new stations anyway. # self._info('Scanning folder ' + folder + ' for stations') + if not 'infos' in options.keys(): + options['infos'] = {} + if not 'short_name' in options['infos'].keys(): + options['infos']['short_name'] = '[name]' + files = os.listdir(folder) for file in files: filepath = os.path.join(folder, file) @@ -205,6 +210,7 @@ def create_station(self, folder, options): if not 'media' in s.keys(): s['media'] = {} s['media']['source'] = folder + self.add_station(s) def load_stations_fromconfig(self, folder): diff --git a/deefuzzer/station.py b/deefuzzer/station.py index 566b8e5..5804866 100644 --- a/deefuzzer/station.py +++ b/deefuzzer/station.py @@ -74,7 +74,6 @@ class Station(Thread): relay_mode = 0 record_mode = 0 run_mode = 1 - station_dir = None appendtype = 1 feeds_json = 0 feeds_rss = 1 @@ -87,6 +86,7 @@ class Station(Thread): starting_id = -1 jingles_frequency = 2 statusfile = '' + base_directory = '' def __init__(self, station, q, logqueue, m3u): Thread.__init__(self) @@ -105,21 +105,21 @@ def __init__(self, station, q, logqueue, m3u): except: pass - if 'station_dir' in self.station: - self.station_dir = self.station['station_dir'] - + if 'base_dir' in self.station: + self.base_directory = self.station['base_dir'].strip() + # Media if 'm3u' in self.station['media'].keys(): if not self.station['media']['m3u'].strip() == '': - self.media_source = self.station['media']['m3u'] + self.media_source = self._path_add_base(self.station['media']['m3u']) if 'dir' in self.station['media'].keys(): if not self.station['media']['dir'].strip() == '': - self.media_source = self.station['media']['dir'] + self.media_source = self._path_add_base(self.station['media']['dir']) if 'source' in self.station['media'].keys(): if not self.station['media']['source'].strip() == '': - self.media_source = self.station['media']['source'] + self.media_source = self._path_add_base(self.station['media']['source']) self.media_format = self.station['media']['format'] self.shuffle_mode = int(self.station['media']['shuffle']) @@ -188,7 +188,7 @@ def __init__(self, station, q, logqueue, m3u): if 'rss' in self.station: if 'mode' in self.station['rss']: self.feeds_mode = int(self.station['rss']['mode']) - self.feeds_dir = self.station['rss']['dir'] + self.feeds_dir = self._path_add_base(self.station['rss']['dir']) self.feeds_enclosure = int(self.station['rss']['enclosure']) if 'json' in self.station['rss']: self.feeds_json = int(self.station['rss']['json']) @@ -248,7 +248,7 @@ def __init__(self, station, q, logqueue, m3u): if 'frequency' in self.station['jingles']: self.jingles_frequency = int(self.station['jingles']['frequency']) if 'dir' in self.station['jingles']: - self.jingles_dir = self.station['jingles']['dir'] + self.jingles_dir = self._path_add_base(self.station['jingles']['dir']) if self.jingles_mode == 1: self.jingles_callback('/jingles', [1]) @@ -279,12 +279,15 @@ def __init__(self, station, q, logqueue, m3u): # Recording if 'record' in self.station: self.record_mode = int(self.station['record']['mode']) - self.record_dir = self.station['record']['dir'] + self.record_dir = self._path_add_base(self.station['record']['dir']) if self.record_mode: self.record_callback('/record', [1]) self.valid = True - + + def _path_add_base(self, a): + return os.path.join(self.base_directory, a) + def _log(self, level, msg): try: obj = {} diff --git a/example/deefuzzer_doc.xml b/example/deefuzzer_doc.xml index 93dcc3e..193e9b5 100644 --- a/example/deefuzzer_doc.xml +++ b/example/deefuzzer_doc.xml @@ -39,6 +39,18 @@ + + /path/to/station/folder @@ -171,9 +183,7 @@ 1 - @@ -181,9 +191,6 @@ [name] [name] - - [path] - From f53dfef8644c25b5896243b0149d838ec280e175 Mon Sep 17 00:00:00 2001 From: achbed Date: Sat, 24 Jan 2015 12:14:13 -0600 Subject: [PATCH 50/82] Fix for #51 (M3U input does not support relative paths) Signed-off-by: achbed --- deefuzzer/station.py | 6 +++++- example/deefuzzer_doc.xml | 1 - 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/deefuzzer/station.py b/deefuzzer/station.py index f558e4e..8f67229 100644 --- a/deefuzzer/station.py +++ b/deefuzzer/station.py @@ -288,6 +288,9 @@ def __init__(self, station, q, logqueue, m3u): def _path_add_base(self, a): return os.path.join(self.base_directory, a) + def _path_m3u_rel(self, a): + return os.path.join(os.path.dirname(self.source), a) + def _log(self, level, msg): try: obj = {} @@ -419,8 +422,9 @@ def get_playlist(self): f = open(self.media_source, 'r') try: for path in f.readlines(): + path = path.strip() if '#' != path[0]: - fp = self._path_add_base(path.strip()) + fp = self._path_m3u_rel(path) if os.path.isfile(fp): file_list.append(fp) except: diff --git a/example/deefuzzer_doc.xml b/example/deefuzzer_doc.xml index e24c8ea..20b68d0 100644 --- a/example/deefuzzer_doc.xml +++ b/example/deefuzzer_doc.xml @@ -48,7 +48,6 @@ - media/source (except for stations created using the stationfolder method) - record/dir - feeds/dir - - All paths read from the file specified by media/m3u or media/source If the parameter path begins with a slash, it is assumed to be an absolute path and prepending does not occur (see https://docs.python.org/2/library/os.path.html#os.path.join ) From ef20d2e35a656e21376ba3942aaf2468b151296b Mon Sep 17 00:00:00 2001 From: achbed Date: Sat, 24 Jan 2015 12:57:25 -0600 Subject: [PATCH 51/82] Playlist feeds now recreated at track change time Fix for #52 --- deefuzzer/station.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/deefuzzer/station.py b/deefuzzer/station.py index 8f67229..9ab3ab6 100644 --- a/deefuzzer/station.py +++ b/deefuzzer/station.py @@ -492,9 +492,6 @@ def get_next_media(self): self._info('Generating new playlist (' + str(self.lp) + ' tracks)') - if self.feeds_playlist: - self.update_feeds(self.media_to_objs(self.playlist), self.feeds_playlist_file, '(playlist)') - elif lp_new != self.lp: self.id += 1 if self.id >= lp_new: @@ -521,10 +518,8 @@ def get_next_media(self): self.playlist = playlist - self._info('generating new playlist (' + str(self.lp) + ' tracks)') + self._info('Generating new playlist (' + str(self.lp) + ' tracks)') - if self.feeds_playlist: - self.update_feeds(self.media_to_objs(self.playlist), self.feeds_playlist_file, '(playlist)') if self.jingles_mode and not (self.counter % self.jingles_frequency) and self.jingles_length: media = self.jingles_list[self.jingle_id] @@ -538,6 +533,8 @@ def get_next_media(self): f = open(self.statusfile, 'w') f.write(str(self.id)) f.close() + if self.feeds_playlist: + self.update_feeds(self.media_to_objs(self.playlist), self.feeds_playlist_file, '(playlist)') except: pass self.q.task_done() From 29fa4af81d32f1c7bfc68297343b3d6ec49dd3f6 Mon Sep 17 00:00:00 2001 From: Guillaume Pellerin Date: Sat, 24 Jan 2015 22:31:43 +0100 Subject: [PATCH 52/82] add dep to pyyaml --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 56cccad..bbfc97c 100644 --- a/setup.py +++ b/setup.py @@ -25,6 +25,7 @@ 'mutagen', 'pyliblo', 'pycurl', + 'pyyaml', ], platforms=['OS Independent'], license='CeCILL v2', From 5aa64a3955ab5fdbed9d9a8a60d74ff8d5c148cb Mon Sep 17 00:00:00 2001 From: Guillaume Pellerin Date: Sun, 25 Jan 2015 00:43:01 +0100 Subject: [PATCH 53/82] update travis setup --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index a8bc11e..2ac2879 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,8 +6,8 @@ virtualenv: # command to prepare the system to install prerequisites or dependencies before_install: - - sudo apt-get install python-setuptools - - sudo apt-get install -qq libshout3-dev python-liblo python-mutagen python-pycurl + - sudo apt-get install -qq python-setuptools + - sudo apt-get install -qq python-pip python-dev libshout3-dev python-liblo python-mutagen python-pycurl liblo-dev libshout3-dev librtmp-dev python-yaml libcurl4-openssl-dev python-mutagen # command to run tests script: From c32120d1dd7302fe00dab2f948de5fb925888085 Mon Sep 17 00:00:00 2001 From: Guillaume Pellerin Date: Sun, 25 Jan 2015 01:41:48 +0100 Subject: [PATCH 54/82] README: update news and other things --- README.rst | 104 ++++++++++++++++++++++++++++++----------------------- 1 file changed, 60 insertions(+), 44 deletions(-) diff --git a/README.rst b/README.rst index e9715d4..8588ef0 100644 --- a/README.rst +++ b/README.rst @@ -1,13 +1,26 @@ .. image:: https://github.com/yomguy/DeeFuzzer/raw/master/doc/img/logo_deefuzzer.png -Introduction -============ +|version| |downloads| |travis_master| + +.. |travis_master| image:: https://secure.travis-ci.org/Parisson/DeeFuzzer.png?branch=master + :target: https://travis-ci.org/Parisson/DeeFuzzer/ + +.. |version| image:: https://pypip.in/version/DeeFuzzer/badge.png + :target: https://pypi.python.org/pypi/DeeFuzzer/ + :alt: Version + +.. |downloads| image:: https://pypip.in/download/DeeFuzzer/badge.svg + :target: https://pypi.python.org/pypi/DeeFuzzer/ + :alt: Downloads + -DeeFuzzer is an open, light and instant software made for streaming audio and video over Internet. -It is dedicated to media streaming communities who wants to create web radios, web televisions, -live multimedia relaying or even as a personal home radio, with rich media contents including track metadata. +DeeFuzzer is a light and instant application for streaming audio and video over internet. +It is dedicated to communities who wants to easily create web radios, web TVs, +live multimedia relays or personal home radios, with metadata management and cool features. -Here are the main features of the deefuzzer: + +Features +========= * MP3, OGG Vorbis file and live streaming over Internet * Full metadata encapsulation and management @@ -15,23 +28,40 @@ Here are the main features of the deefuzzer: * M3U playlist generator * Recursive, random (shuffled) or pre-defined playlists * Multi-threaded architecture: multiple station streaming with one config file - * Auto Twitter posting of the current playing and new tracks - * Jingling between main tracks + * Auto twitting the current playing track and new tracks + * Auto jingling between tracks * OSC controller: control the main functions from a distant terminal - * Relaying: relay and stream *LIVE* sessions! + * Relaying: relay and stream live sessions! + * WebM video relaying support * Very light and optimized streaming process * Fully written in Python * Works with Icecast2, ShoutCast, Stream-m - * EXPERIMENTAL: WebM video streaming support -It is only neccessary to provide a config file which sets all needed parameters. -Please see example/deefuzzer.xml for an example. +Because our aim is to get DeeFuzzer as light as possible it is NOT capable of re-encoding or transcoding media files for the moment. -Because our aim is to get DeeFuzzer as light as possible it is NOT capable of re-encoding or transcoding media files. News ===== +0.7 + + * Huge refactoring which should be compatible with old setups, but please read the `updated example `_ before going on + * Reworked the RSS feed handling to allow JSON output as well and more configuration options (@achbed #27 #28) + * Add an init.d script to act as a deamon (@achbed) + * Add stationdefaults preference (apply default settings to all stations) (@achbed #31) + * Add stationfolder preference (generate stations automagically from a folder structure) (@achbed #31) + * Add stationconfig preference (load other preference files as stations) (@achbed #31) + * Add new station.server.appendtype option + * Add new base_dir parameter to station definition + * New stationdefaults parameter (provides a mechanism to give default values for all stations) (@achbed #29) + * Better thread management (@achbed #36 #37 #38) + * Improved stability avoiding crashes with automatic station restart methods (@achbed #39 #45) + * Added option (ignoreerrors) to log and continue when an error occurs during station setup (@achbed #43) + * Cleanup, better doucmentation and good ideas (@choiz #15 #16 #17 #23) + * Various bufixes + * Many thanks to all participants and espacially to @achbed for his **huge** work, efficiency and easy collaboration + * Enjoy! + 0.6.6 * Update station name (remove ": http://url") @@ -82,40 +112,23 @@ To install it, say on Debian, do:: sudo apt-get install python-pip python-dev libshout3-dev python-liblo python-mutagen \ python-pycurl liblo-dev libshout3-dev librtmp-dev \ - python-yaml libcurl4-openssl-dev python-mutagen + python-yaml libcurl4-openssl-dev python-mutagen icecast2 Now, the easiest way to install the DeeFuzzer from a shell:: sudo pip install deefuzzer -or:: - - sudo easy_install deefuzzer - -to upgrade:: +To upgrade:: - sudo pip install --upgrade deefuzzer + sudo pip install -U deefuzzer -To install the DeeFuzzer from sources, download the last archive `there `_ +For more informations, please see on `GitHub `_ or tweet to @yomguy -Uncompress, go to the deefuzzer app directory and run install as root. For example:: - - tar xzf DeeFuzzer-0.6.5.tar.gz - cd DeeFuzzer-0.6.5 - sudo python setup.py install - -Follow the related package list to install optional or recommended applications: - - * **depends**: python, python-dev, python-shout (from pypi.python.org) | shout-python, libshout3, libshout3-dev, python-mutagen, python-pycurl | pycurl - * **optional**: python-twitter, python-liblo | pyliblo (>= 0.26), python-yaml - * **recommends**: icecast2, python-setuptools, stream-m - -For more informations, please see on `GitHub `_ or twitt a message to @parisson_studio Usage ===== -Usage: deefuzzer CONFIGFILE +deefuzzer CONFIGFILE where CONFIGFILE is the path for a XML or YAML config file. For example:: @@ -132,7 +145,7 @@ To make the deefuzzer act as a deamon, just play it in the background:: Note that you must edit the config file with right parameters before playing. You can find an example for a draft XML file in the "example" directory of the source code. -WARNING: because we need the DeeFuzer to be a very stable streaming process with multiple channel management, +WARNING: because we need the DeeFuzzer to be a very stable streaming process with multiple channel management, the multi-threaded implementation of deefuzzer instances avoids shutting down the process with a CTRL+C. You have to kill them manually, after a CTRL+Z, making this:: @@ -148,8 +161,8 @@ Configuration Some examples of markup configuration files: - * `generic XML `_ * `generic and documented XML `_ + * `generic XML for testing `_ * `OGG Vorbis and MP3 together `_ * `generic YAML `_ @@ -260,22 +273,23 @@ API http://files.parisson.com/doc/deefuzzer/ + Development ============ Everybody is welcome to participate to the DeeFuzzer project! We use GitHub to collaborate: https://github.com/yomguy/DeeFuzzer -Join us! +Clone it, star it, join us! -Author -====== -YomguY aka Guillaume Pellerin: +Authors +======= + + * @yomguy +GuillaumePellerin yomguy@parisson.com + * @achbed + * @choiz - * twitter @yomguy @parisson_studio - * g+ +Guillaume Pellerin - * email License ======= @@ -283,6 +297,7 @@ License This software is released under the terms of the CeCILL license (GPLv2 compatible). as described in the file LICENSE.txt in the source directory or online https://github.com/yomguy/DeeFuzzer/blob/master/LICENSE.txt + Aknowledgements =============== @@ -294,6 +309,7 @@ from scratch in python. Some parts of this work are also taken from another Parisson's project: Telemeta (see http://telemeta.org). + Contact / Infos =============== From ea5692d51ce1ab74ede69645101fc740c6f0f2ef Mon Sep 17 00:00:00 2001 From: Guillaume Pellerin Date: Sun, 25 Jan 2015 01:43:04 +0100 Subject: [PATCH 55/82] fix pins --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 8588ef0..b898a9a 100644 --- a/README.rst +++ b/README.rst @@ -2,8 +2,8 @@ |version| |downloads| |travis_master| -.. |travis_master| image:: https://secure.travis-ci.org/Parisson/DeeFuzzer.png?branch=master - :target: https://travis-ci.org/Parisson/DeeFuzzer/ +.. |travis_master| image:: https://secure.travis-ci.org/yomguy/DeeFuzzer.png?branch=master + :target: https://travis-ci.org/yomguy/DeeFuzzer/ .. |version| image:: https://pypip.in/version/DeeFuzzer/badge.png :target: https://pypi.python.org/pypi/DeeFuzzer/ From 095d4dbbfe7dae40dc1c4b24ad50bb46dbdc8d95 Mon Sep 17 00:00:00 2001 From: Guillaume Pellerin Date: Sun, 25 Jan 2015 01:50:53 +0100 Subject: [PATCH 56/82] Move old news --- NEWS.rst | 29 +++++++++++++++++++++++++++++ README.rst | 26 +------------------------- 2 files changed, 30 insertions(+), 25 deletions(-) create mode 100644 NEWS.rst diff --git a/NEWS.rst b/NEWS.rst new file mode 100644 index 0000000..602c6d2 --- /dev/null +++ b/NEWS.rst @@ -0,0 +1,29 @@ +Old news +========= + +See README.rst for last news. + +0.6.4 + + * Fix install bug again (add main script to install), sorry :( + * Reduce streaming buffer length + +0.6.3 + + * Fix install bug + * Setup rewritten + * Fix MANIFEST + +0.6.2 + + * No new functions but bugfixes (including a serious one during install from pypi) + * Definitely moved the project to `GitHub `_ + * Update various README details + * update API doc: http://files.parisson.com/doc/deefuzzer/ + +0.6.1 + + * New HTTP steamer based on pycurl to handle streaming over stream-m servers (WebM streaming) + see http://code.google.com/p/stream-m/ + * Live webm relaying works good, webm playlist reading NEED testing + * New parameter ('icecast or 'stream-m') diff --git a/README.rst b/README.rst index b898a9a..04b2e7e 100644 --- a/README.rst +++ b/README.rst @@ -59,7 +59,7 @@ News * Added option (ignoreerrors) to log and continue when an error occurs during station setup (@achbed #43) * Cleanup, better doucmentation and good ideas (@choiz #15 #16 #17 #23) * Various bufixes - * Many thanks to all participants and espacially to @achbed for his **huge** work, efficiency and easy collaboration + * Many thanks to all participants and especially to @achbed for his **huge** work, efficiency and easy collaboration * Enjoy! 0.6.6 @@ -76,30 +76,6 @@ News * Read m3u playlist files * Minor fixes -0.6.4 is out! - - * Fix install bug again (add main script to install), sorry :( - * Reduce streaming buffer length - -0.6.3 Fix install bug! - - * Setup rewritten - * Fix MANIFEST - -0.6.2 has been released! - - * No new functions but bugfixes (including a serious one during install from pypi) - * Definitely moved the project to `GitHub `_ - * Update various README details - * update API doc: http://files.parisson.com/doc/deefuzzer/ - -0.6.1 is out! - - * New HTTP steamer based on pycurl to handle streaming over stream-m servers (WebM streaming) - see http://code.google.com/p/stream-m/ - * Live webm relaying works good, webm playlist reading NEED testing - * New parameter ('icecast or 'stream-m') - Installation ============ From 052001bb015476fa490a5af32f8a134617d20ae9 Mon Sep 17 00:00:00 2001 From: achbed Date: Sun, 25 Jan 2015 14:59:43 -0600 Subject: [PATCH 57/82] Added exclusion for PyCharm IDE files --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 885eeb7..f329d8f 100644 --- a/.gitignore +++ b/.gitignore @@ -26,3 +26,6 @@ pip-log.txt #Mr Developer .mr.developer.cfg + +#PyCharm IDE +.idea From 9b5b163243da4d19ff4de92b0993ac37d22ec418 Mon Sep 17 00:00:00 2001 From: achbed Date: Sun, 25 Jan 2015 15:40:13 -0600 Subject: [PATCH 58/82] Whitespace and formatting fixes to better meet PEP8 Coding Style Signed-off-by: achbed --- README.rst | 8 +-- deefuzzer/core.py | 10 ++- deefuzzer/station.py | 161 +++++++++++++++++++++++-------------------- scripts/deefuzzer | 2 + 4 files changed, 97 insertions(+), 84 deletions(-) diff --git a/README.rst b/README.rst index e9715d4..9f877c2 100644 --- a/README.rst +++ b/README.rst @@ -30,7 +30,7 @@ Please see example/deefuzzer.xml for an example. Because our aim is to get DeeFuzzer as light as possible it is NOT capable of re-encoding or transcoding media files. News -===== +==== 0.6.6 @@ -144,7 +144,7 @@ or, more specificially:: Configuration -============== +============= Some examples of markup configuration files: @@ -178,7 +178,7 @@ from a console or any application (see deefuzzer/scripts/). Twitter (manual and optional) -================================ +============================= To get track twitting, please install python-twitter, python-oauth2 and all their dependencies. @@ -261,7 +261,7 @@ API http://files.parisson.com/doc/deefuzzer/ Development -============ +=========== Everybody is welcome to participate to the DeeFuzzer project! We use GitHub to collaborate: https://github.com/yomguy/DeeFuzzer diff --git a/deefuzzer/core.py b/deefuzzer/core.py index ab95474..7fbaf07 100644 --- a/deefuzzer/core.py +++ b/deefuzzer/core.py @@ -118,9 +118,7 @@ def __init__(self, conf_file): def _log(self, level, msg): try: - obj = {} - obj['msg'] = 'Core: ' + str(msg) - obj['level'] = level + obj = {'msg': 'Core: ' + str(msg), 'level': level} self.logqueue.put(obj) except: pass @@ -295,8 +293,8 @@ def run(self): self._err('Station ' + name + ' is stopped and will not be restarted.') self.station_settings[i]['station_stop_logged'] = True continue - - self.station_settings[i]['retries'] = self.station_settings[i]['retries'] + 1 + + self.station_settings[i]['retries'] += 1 self._info('Restarting station ' + name + ' (try ' + str(self.station_settings[i]['retries']) + ')') except Exception as e: self._err('Error checking status for ' + name) @@ -316,7 +314,7 @@ def run(self): name = self.station_settings[i]['infos']['short_name'] y = 1 while name in self.station_instances.keys(): - y = y + 1 + y += 1 name = self.station_settings[i]['infos']['short_name'] + " " + str(y) self.station_settings[i]['station_name'] = name diff --git a/deefuzzer/station.py b/deefuzzer/station.py index 9ab3ab6..36124dd 100644 --- a/deefuzzer/station.py +++ b/deefuzzer/station.py @@ -99,7 +99,7 @@ def __init__(self, station, q, logqueue, m3u): self.statusfile = station['station_statusfile'] try: if os.path.exists(self.statusfile): - f = open(self.statusfile,'r') + f = open(self.statusfile, 'r') self.starting_id = int(f.read()) f.close() except: @@ -107,20 +107,20 @@ def __init__(self, station, q, logqueue, m3u): if 'base_dir' in self.station: self.base_directory = self.station['base_dir'].strip() - + # Media if 'm3u' in self.station['media'].keys(): if not self.station['media']['m3u'].strip() == '': self.media_source = self._path_add_base(self.station['media']['m3u']) - + if 'dir' in self.station['media'].keys(): if not self.station['media']['dir'].strip() == '': self.media_source = self._path_add_base(self.station['media']['dir']) - + if 'source' in self.station['media'].keys(): if not self.station['media']['source'].strip() == '': self.media_source = self._path_add_base(self.station['media']['source']) - + self.media_format = self.station['media']['format'] self.shuffle_mode = int(self.station['media']['shuffle']) self.bitrate = int(self.station['media']['bitrate']) @@ -142,7 +142,7 @@ def __init__(self, station, q, logqueue, m3u): self.appendtype = int(self.station['server']['appendtype']) if 'type' in self.station['server']: - self.type = self.station['server']['type'] # 'icecast' | 'stream-m' + self.type = self.station['server']['type'] # 'icecast' | 'stream-m' else: self.type = 'icecast' @@ -169,14 +169,14 @@ def __init__(self, station, q, logqueue, m3u): self.channel.password = self.station['server']['sourcepassword'] self.channel.public = int(self.station['server']['public']) if self.channel.format == 'mp3': - self.channel.audio_info = { 'bitrate': str(self.bitrate), - 'samplerate': str(self.samplerate), - 'channels': str(self.voices),} + self.channel.audio_info = {'bitrate': str(self.bitrate), + 'samplerate': str(self.samplerate), + 'channels': str(self.voices), } else: - self.channel.audio_info = { 'bitrate': str(self.bitrate), - 'samplerate': str(self.samplerate), - 'quality': str(self.ogg_quality), - 'channels': str(self.voices),} + self.channel.audio_info = {'bitrate': str(self.bitrate), + 'samplerate': str(self.samplerate), + 'quality': str(self.ogg_quality), + 'channels': str(self.voices), } self.server_url = 'http://' + self.channel.host + ':' + str(self.channel.port) self.channel_url = self.server_url + self.channel.mount @@ -242,7 +242,7 @@ def __init__(self, station, q, logqueue, m3u): # Jingling between each media. if 'jingles' in self.station: if 'mode' in self.station['jingles']: - self.jingles_mode = int(self.station['jingles']['mode']) + self.jingles_mode = int(self.station['jingles']['mode']) if 'shuffle' in self.station['jingles']: self.jingles_shuffle = int(self.station['jingles']['shuffle']) if 'frequency' in self.station['jingles']: @@ -268,7 +268,7 @@ def __init__(self, station, q, logqueue, m3u): self.twitter_tags = self.station['twitter']['tags'].split(' ') try: self.twitter_messages = self.station['twitter']['message'] - if isinstance(self.twitter_messages, dict): + if isinstance(self.twitter_messages, dict): self.twitter_messages = list(self.twitter_messages) except: pass @@ -284,18 +284,16 @@ def __init__(self, station, q, logqueue, m3u): self.record_callback('/record', [1]) self.valid = True - + def _path_add_base(self, a): return os.path.join(self.base_directory, a) - + def _path_m3u_rel(self, a): return os.path.join(os.path.dirname(self.source), a) - + def _log(self, level, msg): try: - obj = {} - obj['msg'] = 'Station ' + str(self.channel_url) + ': ' + str(msg) - obj['level'] = str(level) + obj = {'msg': 'Station ' + str(self.channel_url) + ': ' + str(msg), 'level': str(level)} self.logqueue.put(obj) except: pass @@ -341,7 +339,7 @@ def twitter_callback(self, path, value): self.twitter = Twitter(self.twitter_key, self.twitter_secret) self.twitter_mode = value message = "received OSC message '%s' with arguments '%d'" % (path, value) - + # IMPROVEMENT: The URL paths should be configurable because they're # server-implementation specific self.m3u_url = self.channel.url + '/m3u/' + self.m3u.split(os.sep)[-1] @@ -363,9 +361,9 @@ def record_callback(self, path, value): if value: if not os.path.exists(self.record_dir): os.makedirs(self.record_dir) - self.rec_file = self.short_name.replace('/', '_') + '-' + \ - datetime.datetime.now().strftime("%x-%X").replace('/', '_') + \ - '.' + self.channel.format + self.rec_file = self.short_name.replace('/', '_') + '-' + self.rec_file += datetime.datetime.now().strftime("%x-%X").replace('/', '_') + self.rec_file += '.' + self.channel.format self.recorder = Recorder(self.record_dir) self.recorder.open(self.rec_file) else: @@ -382,10 +380,10 @@ def record_callback(self, path, value): if self.channel.format == 'ogg': media = Ogg(self.record_dir + os.sep + self.rec_file) media.metadata = {'artist': self.artist.encode('utf-8'), - 'title': self.title.encode('utf-8'), - 'album': self.short_name.encode('utf-8'), - 'genre': self.channel.genre.encode('utf-8'), - 'date' : date.encode('utf-8'),} + 'title': self.title.encode('utf-8'), + 'album': self.short_name.encode('utf-8'), + 'genre': self.channel.genre.encode('utf-8'), + 'date': date.encode('utf-8'), } media.write_tags() self.record_mode = value @@ -400,7 +398,7 @@ def player_callback(self, path, value): def get_playlist(self): file_list = [] - + try: if os.path.isdir(self.media_source): self.q.get(1) @@ -408,14 +406,14 @@ def get_playlist(self): for root, dirs, files in os.walk(self.media_source): for file in files: s = file.split('.') - ext = s[len(s)-1] - if ext.lower() == self.channel.format and not os.sep+'.' in file: + ext = s[len(s) - 1] + if ext.lower() == self.channel.format and not os.sep + '.' in file: file_list.append(root + os.sep + file) file_list.sort() except: pass self.q.task_done() - + if os.path.isfile(self.media_source): self.q.get(1) try: @@ -434,7 +432,7 @@ def get_playlist(self): self.q.task_done() except: pass - + return file_list def get_jingles(self): @@ -442,8 +440,8 @@ def get_jingles(self): for root, dirs, files in os.walk(self.jingles_dir): for file in files: s = file.split('.') - ext = s[len(s)-1] - if ext.lower() == self.channel.format and not os.sep+'.' in file: + ext = s[len(s) - 1] + if ext.lower() == self.channel.format and not os.sep + '.' in file: file_list.append(root + os.sep + file) file_list.sort() return file_list @@ -466,9 +464,9 @@ def tweet(self): artist = artist.encode('utf-8') artist_names = artist.split(' ') - artist_tags = ' #'.join(list(set(artist_names)-set(['&', '-']))) + artist_tags = ' #'.join(list(set(artist_names) - {'&', '-'})) message = '#NEWTRACK ! %s #%s on #%s RSS: ' % \ - (song.replace('_', ' '), artist_tags, self.short_name) + (song.replace('_', ' '), artist_tags, self.short_name) message = message[:113] + self.feeds_url self.update_twitter(message) @@ -481,7 +479,7 @@ def get_next_media(self): if not self.counter: self.id = 0 - if self.starting_id > -1 and self.starting_id < lp_new: + if -1 < self.starting_id < lp_new: self.id = self.starting_id self.playlist = new_playlist self.lp = lp_new @@ -505,7 +503,7 @@ def get_next_media(self): new_tracks = new_playlist_set - playlist_set self.new_tracks = list(new_tracks.copy()) - if self.twitter_mode == 1 and self.counter: + if self.twitter_mode == 1 and self.counter: self.tweet() # Shake it, Fuzz it ! @@ -520,7 +518,6 @@ def get_next_media(self): self._info('Generating new playlist (' + str(self.lp) + ' tracks)') - if self.jingles_mode and not (self.counter % self.jingles_frequency) and self.jingles_length: media = self.jingles_list[self.jingle_id] self.jingle_id = (self.jingle_id + 1) % self.jingles_length @@ -566,9 +563,9 @@ def media_to_objs(self, media_list): except: continue if self.feeds_showfilename: - file_meta.metadata['filename'] = file_name.decode( "utf-8" ) #decode needed for some weird filenames + file_meta.metadata['filename'] = file_name.decode("utf-8") # decode needed for some weird filenames if self.feeds_showfilepath: - file_meta.metadata['filepath'] = media.decode( "utf-8" ) #decode needed for some weird filenames + file_meta.metadata['filepath'] = media.decode("utf-8") # decode needed for some weird filenames except: pass self.q.task_done() @@ -620,13 +617,13 @@ def update_feeds(self, media_list, rss_file, sub_title): media_link = self.feeds_media_url + media.file_name media_link = media_link.decode('utf-8') rss_item_list.append(RSSItem( - title = song, - link = media_link, - description = media_description, - enclosure = Enclosure(media_link, str(media.size), 'audio/mpeg'), - guid = Guid(media_link), - pubDate = media_date,) - ) + title=song, + link=media_link, + description=media_description, + enclosure=Enclosure(media_link, str(media.size), 'audio/mpeg'), + guid=Guid(media_link), + pubDate=media_date, ) + ) else: media_link = self.metadata_url + '/' + media.file_name + '.xml' try: @@ -634,19 +631,19 @@ def update_feeds(self, media_list, rss_file, sub_title): except: continue rss_item_list.append(RSSItem( - title = song, - link = media_link, - description = media_description, - guid = Guid(media_link), - pubDate = media_date,) - ) + title=song, + link=media_link, + description=media_description, + guid=Guid(media_link), + pubDate=media_date, ) + ) json_data.append(json_item) - rss = RSS2(title = channel_subtitle, \ - link = self.channel.url, \ - description = self.channel.description.decode('utf-8'), \ - lastBuildDate = date_now, \ - items = rss_item_list,) + rss = RSS2(title=channel_subtitle, + link=self.channel.url, + description=self.channel.description.decode('utf-8'), + lastBuildDate=date_now, + items=rss_item_list, ) self.q.get(1) try: if self.feeds_rss: @@ -659,7 +656,7 @@ def update_feeds(self, media_list, rss_file, sub_title): try: if self.feeds_json: f = open(rss_file + '.json', 'w') - f.write(json.dumps(json_data, separators=(',',':'))) + f.write(json.dumps(json_data, separators=(',', ':'))) f.close() except: pass @@ -710,8 +707,8 @@ def set_read_mode(self): self.artist = self.artist.encode('utf-8') self.metadata_file = self.metadata_dir + os.sep + self.current_media_obj[0].file_name + '.xml' self.update_feeds(self.current_media_obj, self.feeds_current_file, '(currently playing)') - self._info('DeeFuzzing: id = %s, name = %s' \ - % (self.id, self.current_media_obj[0].file_name)) + self._info('DeeFuzzing: id = %s, name = %s' + % (self.id, self.current_media_obj[0].file_name)) self.player.set_media(self.media) self.q.get(1) @@ -728,8 +725,10 @@ def set_webm_read_mode(self): self.channel.set_callback(FileReader(self.media).read_callback) def update_twitter_current(self): + if not self.__twitter_should_update(): + return artist_names = self.artist.split(' ') - artist_tags = ' #'.join(list(set(artist_names)-set(['&', '-']))) + artist_tags = ' #'.join(list(set(artist_names) - {'&', '-'})) message = '%s %s on #%s' % (self.prefix, self.song, self.short_name) tags = '#' + ' #'.join(self.twitter_tags) message = message + ' ' + tags @@ -748,7 +747,7 @@ def channel_open(self): return True except: self._err('channel could not be opened') - + return False def channel_close(self): @@ -781,7 +780,7 @@ def icecastloop_nextmedia(self): self.counter = (self.counter % self.jingles_frequency) + self.jingles_frequency if self.relay_mode: self.set_relay_mode() - elif os.path.exists(self.media) and not os.sep+'.' in self.media: + elif os.path.exists(self.media) and not os.sep + '.' in self.media: if self.lp == 0: self._err('has no media to stream !') return False @@ -792,11 +791,25 @@ def icecastloop_nextmedia(self): self._err('icecastloop_nextmedia: Error: ' + str(e)) return False + def __twitter_should_update(self): + """Returns whether or not an update should be sent to Twitter""" + if not self.twitter_mode: + # Twitter posting disabled. Return false. + return False + + if self.relay_mode: + # We are in relay mode. Return true. + return True + + if self.jingles_mode and (self.counter % self.jingles_frequency): + # We should be playing a jingle, and we don't send jingles to Twiitter. + return False + return True + + def icecastloop_metadata(self): try: - if (not (self.jingles_mode and (self.counter % self.jingles_frequency)) or \ - self.relay_mode) and self.twitter_mode: - self.update_twitter_current() + self.update_twitter_current() self.channel.set_metadata({'song': self.song, 'charset': 'utf-8'}) return True except Exception, e: @@ -833,9 +846,9 @@ def run(self): # first = True for self.chunk in self.stream: # if first: - # first = False + # first = False # else: - # break + # break if self.next_media or not self.run_mode: break @@ -860,7 +873,7 @@ def run(self): self.recorder.close() return try: - self.channel.set_metadata({'song': self.song, 'charset': 'utf8',}) + self.channel.set_metadata({'song': self.song, 'charset': 'utf8', }) self._info('channel restarted') self.channel.send(self.chunk) self.channel.sync() @@ -871,7 +884,7 @@ def run(self): self.recorder.close() return - # send chunk loop end + # send chunk loop end # while run_mode loop end self._info("Play mode ended. Stopping stream.") diff --git a/scripts/deefuzzer b/scripts/deefuzzer index 174dd4d..3c9c6ef 100755 --- a/scripts/deefuzzer +++ b/scripts/deefuzzer @@ -42,6 +42,7 @@ def prog_info(): return desc % (deefuzzer.__version__, platform_system, year) + def main(): if len(sys.argv) >= 2: d = deefuzzer.core.DeeFuzzer(sys.argv[-1]) @@ -50,6 +51,7 @@ def main(): text = prog_info() sys.exit(text) + if __name__ == '__main__': main() From c84739e65a0bf0c9cc2a62c618cf1541efb5b377 Mon Sep 17 00:00:00 2001 From: achbed Date: Sun, 25 Jan 2015 15:40:13 -0600 Subject: [PATCH 59/82] Whitespace and formatting fixes to better meet PEP8 Coding Style Signed-off-by: achbed --- README.rst | 8 +-- deefuzzer/core.py | 79 +++++++++++---------- deefuzzer/station.py | 161 +++++++++++++++++++++++-------------------- scripts/deefuzzer | 2 + 4 files changed, 132 insertions(+), 118 deletions(-) diff --git a/README.rst b/README.rst index e9715d4..9f877c2 100644 --- a/README.rst +++ b/README.rst @@ -30,7 +30,7 @@ Please see example/deefuzzer.xml for an example. Because our aim is to get DeeFuzzer as light as possible it is NOT capable of re-encoding or transcoding media files. News -===== +==== 0.6.6 @@ -144,7 +144,7 @@ or, more specificially:: Configuration -============== +============= Some examples of markup configuration files: @@ -178,7 +178,7 @@ from a console or any application (see deefuzzer/scripts/). Twitter (manual and optional) -================================ +============================= To get track twitting, please install python-twitter, python-oauth2 and all their dependencies. @@ -261,7 +261,7 @@ API http://files.parisson.com/doc/deefuzzer/ Development -============ +=========== Everybody is welcome to participate to the DeeFuzzer project! We use GitHub to collaborate: https://github.com/yomguy/DeeFuzzer diff --git a/deefuzzer/core.py b/deefuzzer/core.py index ab95474..d4ddb0b 100644 --- a/deefuzzer/core.py +++ b/deefuzzer/core.py @@ -46,7 +46,7 @@ from deefuzzer.station import * from deefuzzer.tools import * -mimetypes.add_type('application/x-yaml','.yaml') +mimetypes.add_type('application/x-yaml', '.yaml') class DeeFuzzer(Thread): @@ -68,7 +68,7 @@ def __init__(self, conf_file): self.conf_file = conf_file self.conf = get_conf_dict(self.conf_file) - if not 'deefuzzer' in self.conf.keys(): + if 'deefuzzer' not in self.conf.keys(): return # Get the log setting first (if possible) @@ -115,12 +115,9 @@ def __init__(self, conf_file): self._info('Using libshout version %s' % shout.version()) self._info('Number of stations : ' + str(len(self.station_settings))) - def _log(self, level, msg): try: - obj = {} - obj['msg'] = 'Core: ' + str(msg) - obj['level'] = level + obj = {'msg': 'Core: ' + str(msg), 'level': level} self.logqueue.put(obj) except: pass @@ -139,7 +136,7 @@ def set_m3u_playlist(self): m3u.write('#EXTM3U\n') for k in self.station_instances.keys(): s = self.station_instances[k] - m3u.write('#EXTINF:%s,%s - %s\n' % ('-1',s.short_name, s.channel.name)) + m3u.write('#EXTINF:%s,%s - %s\n' % ('-1', s.short_name, s.channel.name)) m3u.write('http://' + s.channel.host + ':' + str(s.channel.port) + s.channel.mount + '\n') m3u.close() self._info('Writing M3U file to : ' + self.m3u) @@ -148,12 +145,12 @@ def create_stations_fromfolder(self): """Scan a folder for subfolders containing media, and make stations from them all.""" options = self.watchfolder - if not 'folder' in options.keys(): + if 'folder' not in options.keys(): # We have no folder specified. Bail. return if self.mainLoop: - if not 'livecreation' in options.keys(): + if 'livecreation' not in options.keys(): # We have no folder specified. Bail. return @@ -169,14 +166,14 @@ def create_stations_fromfolder(self): # This makes the log file a lot more verbose. Commented out since we report on new stations anyway. # self._info('Scanning folder ' + folder + ' for stations') - if not 'infos' in options.keys(): + if 'infos' not in options.keys(): options['infos'] = {} - if not 'short_name' in options['infos'].keys(): + if 'short_name' not in options['infos'].keys(): options['infos']['short_name'] = '[name]' files = os.listdir(folder) - for file in files: - filepath = os.path.join(folder, file) + for f in files: + filepath = os.path.join(folder, f) if os.path.isdir(filepath): if folder_contains_music(filepath): self.create_station(filepath, options) @@ -184,9 +181,9 @@ def create_stations_fromfolder(self): def station_exists(self, name): try: for s in self.station_settings: - if not 'infos' in s.keys(): + if 'infos' not in s.keys(): continue - if not 'short_name' in s['infos'].keys(): + if 'short_name' not in s['infos'].keys(): continue if s['infos']['short_name'] == name: return True @@ -203,14 +200,14 @@ def create_station(self, folder, options): if self.station_exists(name): return self._info('Creating station for folder ' + folder) - d = dict(path=folder,name=name) + d = dict(path=folder, name=name) for i in options.keys(): - if not 'folder' in i: + if 'folder' not in i: s[i] = replace_all(options[i], d) - if not 'media' in s.keys(): + if 'media' not in s.keys(): s['media'] = {} s['media']['source'] = folder - + self.add_station(s) def load_stations_fromconfig(self, folder): @@ -233,16 +230,16 @@ def load_stations_fromconfig(self, folder): self._info('Loading station config files in ' + folder) files = os.listdir(folder) - for file in files: - filepath = os.path.join(folder, file) + for f in files: + filepath = os.path.join(folder, f) if os.path.isfile(filepath): self.load_station_config(filepath) - def load_station_config(self, file): + def load_station_config(self, f): """Load station configuration(s) from a config file.""" - self._info('Loading station config file ' + file) - stationdef = get_conf_dict(file) + self._info('Loading station config file ' + f) + stationdef = get_conf_dict(f) if isinstance(stationdef, dict): if 'station' in stationdef.keys(): if isinstance(stationdef['station'], dict): @@ -269,18 +266,18 @@ def run(self): while True: self.create_stations_fromfolder() ns_new = len(self.station_settings) - if(ns_new > ns): + if ns_new > ns: self._info('Loading new stations') - + for i in range(0, ns_new): + name = '' try: - name = '' if 'station_name' in self.station_settings[i].keys(): name = self.station_settings[i]['station_name'] - - if not 'retries' in self.station_settings[i].keys(): + + if 'retries' not in self.station_settings[i].keys(): self.station_settings[i]['retries'] = 0 - + try: if 'station_instance' in self.station_settings[i].keys(): # Check for station running here @@ -288,26 +285,28 @@ def run(self): # Station exists and is alive. Don't recreate. self.station_settings[i]['retries'] = 0 continue - + if self.maxretry >= 0 and self.station_settings[i]['retries'] <= self.maxretry: # Station passed the max retries count is will not be reloaded - if not 'station_stop_logged' in self.station_settings[i].keys(): + if 'station_stop_logged' not in self.station_settings[i].keys(): self._err('Station ' + name + ' is stopped and will not be restarted.') self.station_settings[i]['station_stop_logged'] = True continue - - self.station_settings[i]['retries'] = self.station_settings[i]['retries'] + 1 - self._info('Restarting station ' + name + ' (try ' + str(self.station_settings[i]['retries']) + ')') + + self.station_settings[i]['retries'] += 1 + self._info('Restarting station ' + name + ' (try ' + str( + self.station_settings[i]['retries']) + ')') except Exception as e: self._err('Error checking status for ' + name) self._err(str(e)) - if not ignoreErrors: + if not self.ignoreErrors: raise # Apply station defaults if they exist if 'stationdefaults' in self.conf['deefuzzer']: if isinstance(self.conf['deefuzzer']['stationdefaults'], dict): - self.station_settings[i] = merge_defaults(self.station_settings[i], self.conf['deefuzzer']['stationdefaults']) + self.station_settings[i] = merge_defaults(self.station_settings[i], + self.conf['deefuzzer']['stationdefaults']) if name == '': name = 'Station ' + str(i) @@ -316,7 +315,7 @@ def run(self): name = self.station_settings[i]['infos']['short_name'] y = 1 while name in self.station_instances.keys(): - y = y + 1 + y += 1 name = self.station_settings[i]['infos']['short_name'] + " " + str(y) self.station_settings[i]['station_name'] = name @@ -332,13 +331,13 @@ def run(self): self._err('Error validating station ' + name) except Exception: self._err('Error initializing station ' + name) - if not ignoreErrors: + if not self.ignoreErrors: raise continue if self.m3u: self.set_m3u_playlist() - + ns = ns_new self.mainLoop = True diff --git a/deefuzzer/station.py b/deefuzzer/station.py index 9ab3ab6..36124dd 100644 --- a/deefuzzer/station.py +++ b/deefuzzer/station.py @@ -99,7 +99,7 @@ def __init__(self, station, q, logqueue, m3u): self.statusfile = station['station_statusfile'] try: if os.path.exists(self.statusfile): - f = open(self.statusfile,'r') + f = open(self.statusfile, 'r') self.starting_id = int(f.read()) f.close() except: @@ -107,20 +107,20 @@ def __init__(self, station, q, logqueue, m3u): if 'base_dir' in self.station: self.base_directory = self.station['base_dir'].strip() - + # Media if 'm3u' in self.station['media'].keys(): if not self.station['media']['m3u'].strip() == '': self.media_source = self._path_add_base(self.station['media']['m3u']) - + if 'dir' in self.station['media'].keys(): if not self.station['media']['dir'].strip() == '': self.media_source = self._path_add_base(self.station['media']['dir']) - + if 'source' in self.station['media'].keys(): if not self.station['media']['source'].strip() == '': self.media_source = self._path_add_base(self.station['media']['source']) - + self.media_format = self.station['media']['format'] self.shuffle_mode = int(self.station['media']['shuffle']) self.bitrate = int(self.station['media']['bitrate']) @@ -142,7 +142,7 @@ def __init__(self, station, q, logqueue, m3u): self.appendtype = int(self.station['server']['appendtype']) if 'type' in self.station['server']: - self.type = self.station['server']['type'] # 'icecast' | 'stream-m' + self.type = self.station['server']['type'] # 'icecast' | 'stream-m' else: self.type = 'icecast' @@ -169,14 +169,14 @@ def __init__(self, station, q, logqueue, m3u): self.channel.password = self.station['server']['sourcepassword'] self.channel.public = int(self.station['server']['public']) if self.channel.format == 'mp3': - self.channel.audio_info = { 'bitrate': str(self.bitrate), - 'samplerate': str(self.samplerate), - 'channels': str(self.voices),} + self.channel.audio_info = {'bitrate': str(self.bitrate), + 'samplerate': str(self.samplerate), + 'channels': str(self.voices), } else: - self.channel.audio_info = { 'bitrate': str(self.bitrate), - 'samplerate': str(self.samplerate), - 'quality': str(self.ogg_quality), - 'channels': str(self.voices),} + self.channel.audio_info = {'bitrate': str(self.bitrate), + 'samplerate': str(self.samplerate), + 'quality': str(self.ogg_quality), + 'channels': str(self.voices), } self.server_url = 'http://' + self.channel.host + ':' + str(self.channel.port) self.channel_url = self.server_url + self.channel.mount @@ -242,7 +242,7 @@ def __init__(self, station, q, logqueue, m3u): # Jingling between each media. if 'jingles' in self.station: if 'mode' in self.station['jingles']: - self.jingles_mode = int(self.station['jingles']['mode']) + self.jingles_mode = int(self.station['jingles']['mode']) if 'shuffle' in self.station['jingles']: self.jingles_shuffle = int(self.station['jingles']['shuffle']) if 'frequency' in self.station['jingles']: @@ -268,7 +268,7 @@ def __init__(self, station, q, logqueue, m3u): self.twitter_tags = self.station['twitter']['tags'].split(' ') try: self.twitter_messages = self.station['twitter']['message'] - if isinstance(self.twitter_messages, dict): + if isinstance(self.twitter_messages, dict): self.twitter_messages = list(self.twitter_messages) except: pass @@ -284,18 +284,16 @@ def __init__(self, station, q, logqueue, m3u): self.record_callback('/record', [1]) self.valid = True - + def _path_add_base(self, a): return os.path.join(self.base_directory, a) - + def _path_m3u_rel(self, a): return os.path.join(os.path.dirname(self.source), a) - + def _log(self, level, msg): try: - obj = {} - obj['msg'] = 'Station ' + str(self.channel_url) + ': ' + str(msg) - obj['level'] = str(level) + obj = {'msg': 'Station ' + str(self.channel_url) + ': ' + str(msg), 'level': str(level)} self.logqueue.put(obj) except: pass @@ -341,7 +339,7 @@ def twitter_callback(self, path, value): self.twitter = Twitter(self.twitter_key, self.twitter_secret) self.twitter_mode = value message = "received OSC message '%s' with arguments '%d'" % (path, value) - + # IMPROVEMENT: The URL paths should be configurable because they're # server-implementation specific self.m3u_url = self.channel.url + '/m3u/' + self.m3u.split(os.sep)[-1] @@ -363,9 +361,9 @@ def record_callback(self, path, value): if value: if not os.path.exists(self.record_dir): os.makedirs(self.record_dir) - self.rec_file = self.short_name.replace('/', '_') + '-' + \ - datetime.datetime.now().strftime("%x-%X").replace('/', '_') + \ - '.' + self.channel.format + self.rec_file = self.short_name.replace('/', '_') + '-' + self.rec_file += datetime.datetime.now().strftime("%x-%X").replace('/', '_') + self.rec_file += '.' + self.channel.format self.recorder = Recorder(self.record_dir) self.recorder.open(self.rec_file) else: @@ -382,10 +380,10 @@ def record_callback(self, path, value): if self.channel.format == 'ogg': media = Ogg(self.record_dir + os.sep + self.rec_file) media.metadata = {'artist': self.artist.encode('utf-8'), - 'title': self.title.encode('utf-8'), - 'album': self.short_name.encode('utf-8'), - 'genre': self.channel.genre.encode('utf-8'), - 'date' : date.encode('utf-8'),} + 'title': self.title.encode('utf-8'), + 'album': self.short_name.encode('utf-8'), + 'genre': self.channel.genre.encode('utf-8'), + 'date': date.encode('utf-8'), } media.write_tags() self.record_mode = value @@ -400,7 +398,7 @@ def player_callback(self, path, value): def get_playlist(self): file_list = [] - + try: if os.path.isdir(self.media_source): self.q.get(1) @@ -408,14 +406,14 @@ def get_playlist(self): for root, dirs, files in os.walk(self.media_source): for file in files: s = file.split('.') - ext = s[len(s)-1] - if ext.lower() == self.channel.format and not os.sep+'.' in file: + ext = s[len(s) - 1] + if ext.lower() == self.channel.format and not os.sep + '.' in file: file_list.append(root + os.sep + file) file_list.sort() except: pass self.q.task_done() - + if os.path.isfile(self.media_source): self.q.get(1) try: @@ -434,7 +432,7 @@ def get_playlist(self): self.q.task_done() except: pass - + return file_list def get_jingles(self): @@ -442,8 +440,8 @@ def get_jingles(self): for root, dirs, files in os.walk(self.jingles_dir): for file in files: s = file.split('.') - ext = s[len(s)-1] - if ext.lower() == self.channel.format and not os.sep+'.' in file: + ext = s[len(s) - 1] + if ext.lower() == self.channel.format and not os.sep + '.' in file: file_list.append(root + os.sep + file) file_list.sort() return file_list @@ -466,9 +464,9 @@ def tweet(self): artist = artist.encode('utf-8') artist_names = artist.split(' ') - artist_tags = ' #'.join(list(set(artist_names)-set(['&', '-']))) + artist_tags = ' #'.join(list(set(artist_names) - {'&', '-'})) message = '#NEWTRACK ! %s #%s on #%s RSS: ' % \ - (song.replace('_', ' '), artist_tags, self.short_name) + (song.replace('_', ' '), artist_tags, self.short_name) message = message[:113] + self.feeds_url self.update_twitter(message) @@ -481,7 +479,7 @@ def get_next_media(self): if not self.counter: self.id = 0 - if self.starting_id > -1 and self.starting_id < lp_new: + if -1 < self.starting_id < lp_new: self.id = self.starting_id self.playlist = new_playlist self.lp = lp_new @@ -505,7 +503,7 @@ def get_next_media(self): new_tracks = new_playlist_set - playlist_set self.new_tracks = list(new_tracks.copy()) - if self.twitter_mode == 1 and self.counter: + if self.twitter_mode == 1 and self.counter: self.tweet() # Shake it, Fuzz it ! @@ -520,7 +518,6 @@ def get_next_media(self): self._info('Generating new playlist (' + str(self.lp) + ' tracks)') - if self.jingles_mode and not (self.counter % self.jingles_frequency) and self.jingles_length: media = self.jingles_list[self.jingle_id] self.jingle_id = (self.jingle_id + 1) % self.jingles_length @@ -566,9 +563,9 @@ def media_to_objs(self, media_list): except: continue if self.feeds_showfilename: - file_meta.metadata['filename'] = file_name.decode( "utf-8" ) #decode needed for some weird filenames + file_meta.metadata['filename'] = file_name.decode("utf-8") # decode needed for some weird filenames if self.feeds_showfilepath: - file_meta.metadata['filepath'] = media.decode( "utf-8" ) #decode needed for some weird filenames + file_meta.metadata['filepath'] = media.decode("utf-8") # decode needed for some weird filenames except: pass self.q.task_done() @@ -620,13 +617,13 @@ def update_feeds(self, media_list, rss_file, sub_title): media_link = self.feeds_media_url + media.file_name media_link = media_link.decode('utf-8') rss_item_list.append(RSSItem( - title = song, - link = media_link, - description = media_description, - enclosure = Enclosure(media_link, str(media.size), 'audio/mpeg'), - guid = Guid(media_link), - pubDate = media_date,) - ) + title=song, + link=media_link, + description=media_description, + enclosure=Enclosure(media_link, str(media.size), 'audio/mpeg'), + guid=Guid(media_link), + pubDate=media_date, ) + ) else: media_link = self.metadata_url + '/' + media.file_name + '.xml' try: @@ -634,19 +631,19 @@ def update_feeds(self, media_list, rss_file, sub_title): except: continue rss_item_list.append(RSSItem( - title = song, - link = media_link, - description = media_description, - guid = Guid(media_link), - pubDate = media_date,) - ) + title=song, + link=media_link, + description=media_description, + guid=Guid(media_link), + pubDate=media_date, ) + ) json_data.append(json_item) - rss = RSS2(title = channel_subtitle, \ - link = self.channel.url, \ - description = self.channel.description.decode('utf-8'), \ - lastBuildDate = date_now, \ - items = rss_item_list,) + rss = RSS2(title=channel_subtitle, + link=self.channel.url, + description=self.channel.description.decode('utf-8'), + lastBuildDate=date_now, + items=rss_item_list, ) self.q.get(1) try: if self.feeds_rss: @@ -659,7 +656,7 @@ def update_feeds(self, media_list, rss_file, sub_title): try: if self.feeds_json: f = open(rss_file + '.json', 'w') - f.write(json.dumps(json_data, separators=(',',':'))) + f.write(json.dumps(json_data, separators=(',', ':'))) f.close() except: pass @@ -710,8 +707,8 @@ def set_read_mode(self): self.artist = self.artist.encode('utf-8') self.metadata_file = self.metadata_dir + os.sep + self.current_media_obj[0].file_name + '.xml' self.update_feeds(self.current_media_obj, self.feeds_current_file, '(currently playing)') - self._info('DeeFuzzing: id = %s, name = %s' \ - % (self.id, self.current_media_obj[0].file_name)) + self._info('DeeFuzzing: id = %s, name = %s' + % (self.id, self.current_media_obj[0].file_name)) self.player.set_media(self.media) self.q.get(1) @@ -728,8 +725,10 @@ def set_webm_read_mode(self): self.channel.set_callback(FileReader(self.media).read_callback) def update_twitter_current(self): + if not self.__twitter_should_update(): + return artist_names = self.artist.split(' ') - artist_tags = ' #'.join(list(set(artist_names)-set(['&', '-']))) + artist_tags = ' #'.join(list(set(artist_names) - {'&', '-'})) message = '%s %s on #%s' % (self.prefix, self.song, self.short_name) tags = '#' + ' #'.join(self.twitter_tags) message = message + ' ' + tags @@ -748,7 +747,7 @@ def channel_open(self): return True except: self._err('channel could not be opened') - + return False def channel_close(self): @@ -781,7 +780,7 @@ def icecastloop_nextmedia(self): self.counter = (self.counter % self.jingles_frequency) + self.jingles_frequency if self.relay_mode: self.set_relay_mode() - elif os.path.exists(self.media) and not os.sep+'.' in self.media: + elif os.path.exists(self.media) and not os.sep + '.' in self.media: if self.lp == 0: self._err('has no media to stream !') return False @@ -792,11 +791,25 @@ def icecastloop_nextmedia(self): self._err('icecastloop_nextmedia: Error: ' + str(e)) return False + def __twitter_should_update(self): + """Returns whether or not an update should be sent to Twitter""" + if not self.twitter_mode: + # Twitter posting disabled. Return false. + return False + + if self.relay_mode: + # We are in relay mode. Return true. + return True + + if self.jingles_mode and (self.counter % self.jingles_frequency): + # We should be playing a jingle, and we don't send jingles to Twiitter. + return False + return True + + def icecastloop_metadata(self): try: - if (not (self.jingles_mode and (self.counter % self.jingles_frequency)) or \ - self.relay_mode) and self.twitter_mode: - self.update_twitter_current() + self.update_twitter_current() self.channel.set_metadata({'song': self.song, 'charset': 'utf-8'}) return True except Exception, e: @@ -833,9 +846,9 @@ def run(self): # first = True for self.chunk in self.stream: # if first: - # first = False + # first = False # else: - # break + # break if self.next_media or not self.run_mode: break @@ -860,7 +873,7 @@ def run(self): self.recorder.close() return try: - self.channel.set_metadata({'song': self.song, 'charset': 'utf8',}) + self.channel.set_metadata({'song': self.song, 'charset': 'utf8', }) self._info('channel restarted') self.channel.send(self.chunk) self.channel.sync() @@ -871,7 +884,7 @@ def run(self): self.recorder.close() return - # send chunk loop end + # send chunk loop end # while run_mode loop end self._info("Play mode ended. Stopping stream.") diff --git a/scripts/deefuzzer b/scripts/deefuzzer index 174dd4d..3c9c6ef 100755 --- a/scripts/deefuzzer +++ b/scripts/deefuzzer @@ -42,6 +42,7 @@ def prog_info(): return desc % (deefuzzer.__version__, platform_system, year) + def main(): if len(sys.argv) >= 2: d = deefuzzer.core.DeeFuzzer(sys.argv[-1]) @@ -50,6 +51,7 @@ def main(): text = prog_info() sys.exit(text) + if __name__ == '__main__': main() From f547ae74070eac19666023e34037db08b7cedb7f Mon Sep 17 00:00:00 2001 From: achbed Date: Mon, 26 Jan 2015 16:31:28 -0600 Subject: [PATCH 60/82] Initial pass of cleanup based on PyCharm Code Inspection Signed-off-by: achbed --- deefuzzer/player.py | 2 + deefuzzer/station.py | 1 - deefuzzer/streamer.py | 8 +- deefuzzer/tools/PyRSS2Gen.py | 151 +++++++++++++++++----------- deefuzzer/tools/get_access_token.py | 97 +++++++++--------- deefuzzer/tools/logger.py | 12 +-- deefuzzer/tools/mp3.py | 75 +++++++------- deefuzzer/tools/ogg.py | 42 ++++---- deefuzzer/tools/osc.py | 2 +- deefuzzer/tools/streamer.py | 8 +- deefuzzer/tools/twitt.py | 2 +- deefuzzer/tools/utils.py | 38 ++++--- deefuzzer/tools/xml2yaml.py | 6 +- deefuzzer/tools/xmltodict.py | 4 +- 14 files changed, 252 insertions(+), 196 deletions(-) diff --git a/deefuzzer/player.py b/deefuzzer/player.py index 043a80e..27bab04 100644 --- a/deefuzzer/player.py +++ b/deefuzzer/player.py @@ -128,6 +128,8 @@ def set_recorder(self, recorder, mode=1): self.recorder = recorder def read_callback(self, size): + chunk = None + try: chunk = self.relay.read(size) except: diff --git a/deefuzzer/station.py b/deefuzzer/station.py index 36124dd..59eeb15 100644 --- a/deefuzzer/station.py +++ b/deefuzzer/station.py @@ -806,7 +806,6 @@ def __twitter_should_update(self): return False return True - def icecastloop_metadata(self): try: self.update_twitter_current() diff --git a/deefuzzer/streamer.py b/deefuzzer/streamer.py index 8cb70af..49d9909 100644 --- a/deefuzzer/streamer.py +++ b/deefuzzer/streamer.py @@ -38,8 +38,8 @@ from threading import Thread -class HTTPStreamer(Thread): +class HTTPStreamer(Thread): protocol = 'http' host = str port = str @@ -58,6 +58,7 @@ class HTTPStreamer(Thread): def __init__(self): Thread.__init__(self) import pycurl + self.curl = pycurl.Curl() def set_callback(self, read_callback): @@ -68,8 +69,9 @@ def delay(self): def open(self): import pycurl - self.uri = self.protocol + '://' + self.host + ':' + str(self.port) + \ - self.mount + '?' + 'password=' + self.password + + self.uri = self.protocol + '://' + self.host + ':' + str(self.port) + self.uri += self.mount + '?' + 'password=' + self.password self.curl.setopt(pycurl.URL, self.uri) self.curl.setopt(pycurl.NOSIGNAL, 1) self.curl.setopt(pycurl.UPLOAD, 1) diff --git a/deefuzzer/tools/PyRSS2Gen.py b/deefuzzer/tools/PyRSS2Gen.py index fc1f1cf..a2122f7 100644 --- a/deefuzzer/tools/PyRSS2Gen.py +++ b/deefuzzer/tools/PyRSS2Gen.py @@ -10,14 +10,18 @@ # Could make this the base class; will need to add 'publish' class WriteXmlMixin: - def write_xml(self, outfile, encoding = "iso-8859-1"): + def __init__(self): + pass + + def write_xml(self, outfile, encoding="iso-8859-1"): from xml.sax import saxutils + handler = saxutils.XMLGenerator(outfile, encoding) handler.startDocument() self.publish(handler) handler.endDocument() - def to_xml(self, encoding = "iso-8859-1"): + def to_xml(self, encoding="iso-8859-1"): try: import cStringIO as StringIO except ImportError: @@ -27,7 +31,7 @@ def to_xml(self, encoding = "iso-8859-1"): return f.getvalue() -def _element(handler, name, obj, d = {}): +def _element(handler, name, obj, d={}): if isinstance(obj, basestring) or obj is None: # special-case handling to make the API easier # to use for the common case. @@ -39,6 +43,7 @@ def _element(handler, name, obj, d = {}): # It better know how to emit the correct XML. obj.publish(handler) + def _opt_element(handler, name, obj): if obj is None: return @@ -51,20 +56,20 @@ def _format_date(dt): Input date must be in GMT. """ # Looks like: - # Sat, 07 Sep 2002 00:00:01 GMT + # Sat, 07 Sep 2002 00:00:01 GMT # Can't use strftime because that's locale dependent # # Isn't there a standard way to do this for Python? The # rfc822 and email.Utils modules assume a timestamp. The # following is based on the rfc822 module. return "%s, %02d %s %04d %02d:%02d:%02d GMT" % ( - ["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"][dt.weekday()], - dt.day, - ["Jan", "Feb", "Mar", "Apr", "May", "Jun", - "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"][dt.month-1], - dt.year, dt.hour, dt.minute, dt.second) + ["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"][dt.weekday()], + dt.day, + ["Jan", "Feb", "Mar", "Apr", "May", "Jun", + "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"][dt.month - 1], + dt.year, dt.hour, dt.minute, dt.second) + - ## # A couple simple wrapper objects for the fields which # take a simple value other than a string. @@ -77,14 +82,17 @@ class IntElement: to text for XML.) """ element_attrs = {} + def __init__(self, name, val): self.name = name self.val = val + def publish(self, handler): handler.startElement(self.name, self.element_attrs) handler.characters(str(self.val)) handler.endElement(self.name) + class DateElement: """implements the 'publish' API for a datetime.datetime @@ -92,26 +100,34 @@ class DateElement: Converts the datetime to RFC 2822 timestamp (4-digit year). """ + def __init__(self, name, dt): self.name = name self.dt = dt + def publish(self, handler): _element(handler, self.name, _format_date(self.dt)) + + #### class Category: """Publish a category element""" - def __init__(self, category, domain = None): + + def __init__(self, category, domain=None): self.category = category self.domain = domain + def publish(self, handler): d = {} if self.domain is not None: d["domain"] = self.domain _element(handler, "category", self.category, d) + class Cloud: """Publish a cloud""" + def __init__(self, domain, port, path, registerProcedure, protocol): self.domain = domain @@ -119,6 +135,7 @@ def __init__(self, domain, port, path, self.path = path self.registerProcedure = registerProcedure self.protocol = protocol + def publish(self, handler): _element(handler, "cloud", None, { "domain": self.domain, @@ -127,18 +144,20 @@ def publish(self, handler): "registerProcedure": self.registerProcedure, "protocol": self.protocol}) + class Image: """Publish a channel Image""" element_attrs = {} + def __init__(self, url, title, link, - width = None, height = None, description = None): + width=None, height=None, description=None): self.url = url self.title = title self.link = link self.width = width self.height = height self.description = description - + def publish(self, handler): handler.startElement("image", self.element_attrs) @@ -150,7 +169,7 @@ def publish(self, handler): if isinstance(width, int): width = IntElement("width", width) _opt_element(handler, "width", width) - + height = self.height if isinstance(height, int): height = IntElement("height", height) @@ -160,15 +179,18 @@ def publish(self, handler): handler.endElement("image") + class Guid: """Publish a guid Defaults to being a permalink, which is the assumption if it's omitted. Hence strings are always permalinks. """ - def __init__(self, guid, isPermaLink = 1): + + def __init__(self, guid, isPermaLink=1): self.guid = guid self.isPermaLink = isPermaLink + def publish(self, handler): d = {} if self.isPermaLink: @@ -177,12 +199,14 @@ def publish(self, handler): d["isPermaLink"] = "false" _element(handler, "guid", self.guid, d) + class TextInput: """Publish a textInput Apparently this is rarely used. """ element_attrs = {} + def __init__(self, title, description, name, link): self.title = title self.description = description @@ -196,37 +220,45 @@ def publish(self, handler): _element(handler, "name", self.name) _element(handler, "link", self.link) handler.endElement("textInput") - + class Enclosure: """Publish an enclosure""" + def __init__(self, url, length, type): self.url = url self.length = length self.type = type + def publish(self, handler): _element(handler, "enclosure", None, {"url": self.url, "length": str(self.length), "type": self.type, - }) + }) + class Source: """Publish the item's original source, used by aggregators""" + def __init__(self, name, url): self.name = name self.url = url + def publish(self, handler): _element(handler, "source", self.name, {"url": self.url}) + class SkipHours: """Publish the skipHours This takes a list of hours, as integers. """ element_attrs = {} + def __init__(self, hours): self.hours = hours + def publish(self, handler): if self.hours: handler.startElement("skipHours", self.element_attrs) @@ -234,14 +266,17 @@ def publish(self, handler): _element(handler, "hour", str(hour)) handler.endElement("skipHours") + class SkipDays: """Publish the skipDays This takes a list of days as strings. """ element_attrs = {} + def __init__(self, days): self.days = days + def publish(self, handler): if self.days: handler.startElement("skipDays", self.element_attrs) @@ -249,41 +284,43 @@ def publish(self, handler): _element(handler, "day", day) handler.endElement("skipDays") + class RSS2(WriteXmlMixin): """The main RSS class. Stores the channel attributes, with the "category" elements under ".categories" and the RSS items under ".items". """ - + rss_attrs = {"version": "2.0"} element_attrs = {} + def __init__(self, title, link, description, - language = None, - copyright = None, - managingEditor = None, - webMaster = None, - pubDate = None, # a datetime, *in* *GMT* - lastBuildDate = None, # a datetime - - categories = None, # list of strings or Category - generator = _generator_name, - docs = "http://blogs.law.harvard.edu/tech/rss", - cloud = None, # a Cloud - ttl = None, # integer number of minutes - - image = None, # an Image - rating = None, # a string; I don't know how it's used - textInput = None, # a TextInput - skipHours = None, # a SkipHours with a list of integers - skipDays = None, # a SkipDays with a list of strings - - items = None, # list of RSSItems - ): + language=None, + copyright=None, + managingEditor=None, + webMaster=None, + pubDate=None, # a datetime, *in* *GMT* + lastBuildDate=None, # a datetime + + categories=None, # list of strings or Category + generator=_generator_name, + docs="http://blogs.law.harvard.edu/tech/rss", + cloud=None, # a Cloud + ttl=None, # integer number of minutes + + image=None, # an Image + rating=None, # a string; I don't know how it's used + textInput=None, # a TextInput + skipHours=None, # a SkipHours with a list of integers + skipDays=None, # a SkipDays with a list of strings + + items=None, # list of RSSItems + ): self.title = title self.link = link self.description = description @@ -294,7 +331,7 @@ def __init__(self, self.webMaster = webMaster self.pubDate = pubDate self.lastBuildDate = lastBuildDate - + if categories is None: categories = [] self.categories = categories @@ -320,7 +357,7 @@ def publish(self, handler): _element(handler, "description", self.description) self.publish_extensions(handler) - + _opt_element(handler, "language", self.language) _opt_element(handler, "copyright", self.copyright) _opt_element(handler, "managingEditor", self.managingEditor) @@ -374,24 +411,24 @@ def publish_extensions(self, handler): # output after the three required fields. pass - - + class RSSItem(WriteXmlMixin): """Publish an RSS Item""" element_attrs = {} + def __init__(self, - title = None, # string - link = None, # url as string - description = None, # string - author = None, # email address as string - categories = None, # list of string or Category - comments = None, # url as string - enclosure = None, # an Enclosure - guid = None, # a unique string - pubDate = None, # a datetime - source = None, # a Source - ): - + title=None, # string + link=None, # url as string + description=None, # string + author=None, # email address as string + categories=None, # list of string or Category + comments=None, # url as string + enclosure=None, # an Enclosure + guid=None, # a unique string + pubDate=None, # a datetime + source=None, # a Source + ): + if title is None and description is None: raise TypeError( "must define at least one of 'title' or 'description'") @@ -421,7 +458,7 @@ def publish(self, handler): if isinstance(category, basestring): category = Category(category) category.publish(handler) - + _opt_element(handler, "comments", self.comments) if self.enclosure is not None: self.enclosure.publish(handler) @@ -434,7 +471,7 @@ def publish(self, handler): if self.source is not None: self.source.publish(handler) - + handler.endElement("item") def publish_extensions(self, handler): diff --git a/deefuzzer/tools/get_access_token.py b/deefuzzer/tools/get_access_token.py index a614173..6801019 100644 --- a/deefuzzer/tools/get_access_token.py +++ b/deefuzzer/tools/get_access_token.py @@ -6,7 +6,7 @@ # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, @@ -15,76 +15,75 @@ # limitations under the License. -import os import sys # parse_qsl moved to urlparse module in v2.6 try: - from urlparse import parse_qsl + from urlparse import parse_qsl except: - from cgi import parse_qsl + from cgi import parse_qsl import oauth2 as oauth REQUEST_TOKEN_URL = 'https://api.twitter.com/oauth/request_token' -ACCESS_TOKEN_URL = 'https://api.twitter.com/oauth/access_token' +ACCESS_TOKEN_URL = 'https://api.twitter.com/oauth/access_token' AUTHORIZATION_URL = 'https://api.twitter.com/oauth/authorize' -SIGNIN_URL = 'https://api.twitter.com/oauth/authenticate' +SIGNIN_URL = 'https://api.twitter.com/oauth/authenticate' -consumer_key = 'ozs9cPS2ci6eYQzzMSTb4g' +consumer_key = 'ozs9cPS2ci6eYQzzMSTb4g' consumer_secret = '1kNEffHgGSXO2gMNTr8HRum5s2ofx3VQnJyfd0es' if consumer_key is None or consumer_secret is None: - print 'You need to edit this script and provide values for the' - print 'consumer_key and also consumer_secret.' - print '' - print 'The values you need come from Twitter - you need to register' - print 'as a developer your "application". This is needed only until' - print 'Twitter finishes the idea they have of a way to allow open-source' - print 'based libraries to have a token that can be used to generate a' - print 'one-time use key that will allow the library to make the request' - print 'on your behalf.' - print '' - sys.exit(1) + print 'You need to edit this script and provide values for the' + print 'consumer_key and also consumer_secret.' + print '' + print 'The values you need come from Twitter - you need to register' + print 'as a developer your "application". This is needed only until' + print 'Twitter finishes the idea they have of a way to allow open-source' + print 'based libraries to have a token that can be used to generate a' + print 'one-time use key that will allow the library to make the request' + print 'on your behalf.' + print '' + sys.exit(1) signature_method_hmac_sha1 = oauth.SignatureMethod_HMAC_SHA1() -oauth_consumer = oauth.Consumer(key=consumer_key, secret=consumer_secret) -oauth_client = oauth.Client(oauth_consumer) +oauth_consumer = oauth.Consumer(key=consumer_key, secret=consumer_secret) +oauth_client = oauth.Client(oauth_consumer) print 'Requesting temp token from Twitter' resp, content = oauth_client.request(REQUEST_TOKEN_URL, 'GET') if resp['status'] != '200': - print 'Invalid respond from Twitter requesting temp token: %s' % resp['status'] + print 'Invalid respond from Twitter requesting temp token: %s' % resp['status'] else: - request_token = dict(parse_qsl(content)) - - print '' - print 'Please visit this Twitter page and retrieve the pincode to be used' - print 'in the next step to obtaining an Authentication Token:' - print '' - print '%s?oauth_token=%s' % (AUTHORIZATION_URL, request_token['oauth_token']) - print '' - - pincode = raw_input('Pincode? ') - - token = oauth.Token(request_token['oauth_token'], request_token['oauth_token_secret']) - token.set_verifier(pincode) - - print '' - print 'Generating and signing request for an access token' - print '' - - oauth_client = oauth.Client(oauth_consumer, token) - resp, content = oauth_client.request(ACCESS_TOKEN_URL, method='POST', body='oauth_verifier=%s' % pincode) - access_token = dict(parse_qsl(content)) - - if resp['status'] != '200': - print 'The request for a Token did not succeed: %s' % resp['status'] - print access_token - else: - print 'Your Twitter Access Token key: %s' % access_token['oauth_token'] - print ' Access Token secret: %s' % access_token['oauth_token_secret'] + request_token = dict(parse_qsl(content)) + print '' + print 'Please visit this Twitter page and retrieve the pincode to be used' + print 'in the next step to obtaining an Authentication Token:' + print '' + print '%s?oauth_token=%s' % (AUTHORIZATION_URL, request_token['oauth_token']) + print '' + + pincode = raw_input('Pincode? ') + + token = oauth.Token(request_token['oauth_token'], request_token['oauth_token_secret']) + token.set_verifier(pincode) + + print '' + print 'Generating and signing request for an access token' + print '' + + oauth_client = oauth.Client(oauth_consumer, token) + resp, content = oauth_client.request(ACCESS_TOKEN_URL, method='POST', body='oauth_verifier=%s' % pincode) + access_token = dict(parse_qsl(content)) + + if resp['status'] != '200': + print 'The request for a Token did not succeed: %s' % resp['status'] + print access_token + else: + print 'Your Twitter Access Token key: %s' % access_token['oauth_token'] + print ' Access Token secret: %s' % access_token['oauth_token_secret'] + print '' diff --git a/deefuzzer/tools/logger.py b/deefuzzer/tools/logger.py index 5bb6570..ae46e2b 100644 --- a/deefuzzer/tools/logger.py +++ b/deefuzzer/tools/logger.py @@ -22,14 +22,15 @@ def write_info(self, message): def write_error(self, message): self.logger.error(message) + class QueueLogger(Thread): """A queue-based logging object""" - + def __init__(self, file, q): Thread.__init__(self) self.logger = Logger(file) self.q = q - + def run(self): while True: try: @@ -37,16 +38,15 @@ def run(self): if not isinstance(msg, dict): self.logger.write_error(str(msg)) else: - if not 'msg' in msg.keys(): + if 'msg' not in msg.keys(): continue - + if 'level' in msg.keys(): if msg['level'] == 'info': self.logger.write_info(msg['msg']) else: self.logger.write_error(msg['msg']) else: - self.logger.write_error(msg['msg']) + self.logger.write_error(msg['msg']) except: pass - \ No newline at end of file diff --git a/deefuzzer/tools/mp3.py b/deefuzzer/tools/mp3.py index 500c37f..fac15a1 100644 --- a/deefuzzer/tools/mp3.py +++ b/deefuzzer/tools/mp3.py @@ -44,10 +44,11 @@ from mutagen import id3 from utils import * -EasyID3.valid_keys["comment"]="COMM::'XXX'" -EasyID3.valid_keys["copyright"]="TCOP::'XXX'" -EasyID3.valid_keys["country"]="TXXX:COUNTRY:'XXX'" -EasyID3.RegisterTXXXKey("country","COUNTRY") +EasyID3.valid_keys["comment"] = "COMM::'XXX'" +EasyID3.valid_keys["copyright"] = "TCOP::'XXX'" +EasyID3.valid_keys["country"] = "TXXX:COUNTRY:'XXX'" +EasyID3.RegisterTXXXKey("country", "COUNTRY") + class Mp3: """A MP3 file object""" @@ -60,30 +61,30 @@ def __init__(self, media): self.bitrate_default = '192' self.cache_dir = os.sep + 'tmp' self.keys2id3 = {'title': 'TIT2', - 'artist': 'TPE1', - 'album': 'TALB', - 'date': 'TDRC', - 'comment': 'COMM', - 'country': 'COUNTRY', - 'genre': 'TCON', - 'copyright': 'TCOP', - } + 'artist': 'TPE1', + 'album': 'TALB', + 'date': 'TDRC', + 'comment': 'COMM', + 'country': 'COUNTRY', + 'genre': 'TCON', + 'copyright': 'TCOP', + } self.mp3 = MP3(self.media, ID3=EasyID3) self.info = self.mp3.info self.bitrate = int(str(self.info.bitrate)[:-3]) - self.length = datetime.timedelta(0,self.info.length) + self.length = datetime.timedelta(0, self.info.length) try: self.metadata = self.get_file_metadata() except: self.metadata = {'title': '', - 'artist': '', - 'album': '', - 'date': '', - 'comment': '', - 'country': '', - 'genre': '', - 'copyright': '', - } + 'artist': '', + 'album': '', + 'date': '', + 'comment': '', + 'country': '', + 'genre': '', + 'copyright': '', + } self.description = self.get_description() self.mime_type = self.get_mime_type() @@ -93,7 +94,7 @@ def __init__(self, media): self.file_ext = self.media_info[2] self.extension = self.get_file_extension() self.size = os.path.getsize(media) - #self.args = self.get_args() + # self.args = self.get_args() def get_format(self): return 'MP3' @@ -127,21 +128,21 @@ def write_tags(self): self.mp3.tags['TIT2'] = id3.TIT2(encoding=2, text=u'text') self.mp3.save() - #media_id3 = id3.ID3(self.media) - #for tag in self.metadata.keys(): - #if tag in self.dub2id3_dict.keys(): - #frame_text = self.dub2id3_dict[tag] - #value = self.metadata[tag] - #frame = mutagen.id3.Frames[frame_text](3,value) - #try: - #media_id3.add(frame) - #except: - #raise IOError('ExporterError: cannot tag "'+tag+'"') - - #try: - #media_id3.save() - #except: - #raise IOError('ExporterError: cannot write tags') + # media_id3 = id3.ID3(self.media) + # for tag in self.metadata.keys(): + # if tag in self.dub2id3_dict.keys(): + # frame_text = self.dub2id3_dict[tag] + # value = self.metadata[tag] + # frame = mutagen.id3.Frames[frame_text](3,value) + # try: + # media_id3.add(frame) + # except: + # raise IOError('ExporterError: cannot tag "'+tag+'"') + + # try: + # media_id3.save() + # except: + # raise IOError('ExporterError: cannot write tags') media = id3.ID3(self.media) media.add(id3.TIT2(encoding=3, text=self.metadata['title'].decode('utf8'))) diff --git a/deefuzzer/tools/ogg.py b/deefuzzer/tools/ogg.py index 9b3a85f..3c196a1 100644 --- a/deefuzzer/tools/ogg.py +++ b/deefuzzer/tools/ogg.py @@ -55,16 +55,16 @@ def __init__(self, media): self.bitrate_default = '192' self.cache_dir = os.sep + 'tmp' self.keys2ogg = {'title': 'title', - 'artist': 'artist', - 'album': 'album', - 'date': 'date', - 'comment': 'comment', - 'genre': 'genre', - 'copyright': 'copyright', - } + 'artist': 'artist', + 'album': 'album', + 'date': 'date', + 'comment': 'comment', + 'genre': 'genre', + 'copyright': 'copyright', + } self.info = self.ogg.info self.bitrate = int(str(self.info.bitrate)[:-3]) - self.length = datetime.timedelta(0,self.info.length) + self.length = datetime.timedelta(0, self.info.length) self.metadata = self.get_file_metadata() self.description = self.get_description() self.mime_type = self.get_mime_type() @@ -74,7 +74,7 @@ def __init__(self, media): self.file_ext = self.media_info[2] self.extension = self.get_file_extension() self.size = os.path.getsize(media) - #self.args = self.get_args() + # self.args = self.get_args() def get_format(self): return 'OGG' @@ -90,7 +90,7 @@ def get_description(self): def get_file_info(self): try: - file_out1, file_out2 = os.popen4('ogginfo "'+self.dest+'"') + file_out1, file_out2 = os.popen4('ogginfo "' + self.dest + '"') info = [] for line in file_out2.readlines(): info.append(clean_word(line[:-1])) @@ -99,8 +99,8 @@ def get_file_info(self): except: raise IOError('ExporterError: file does not exist.') - def set_cache_dir(self,path): - self.cache_dir = path + def set_cache_dir(self, path): + self.cache_dir = path def get_file_metadata(self): metadata = {} @@ -113,19 +113,19 @@ def get_file_metadata(self): def decode(self): try: - os.system('oggdec -o "'+self.cache_dir+os.sep+self.item_id+ - '.wav" "'+self.source+'"') - return self.cache_dir+os.sep+self.item_id+'.wav' + os.system('oggdec -o "' + self.cache_dir + os.sep + self.item_id + + '.wav" "' + self.source + '"') + return self.cache_dir + os.sep + self.item_id + '.wav' except: raise IOError('ExporterError: decoder is not compatible.') def write_tags(self): - #self.ogg.add_tags() + # self.ogg.add_tags() for tag in self.metadata.keys(): self.ogg[tag] = str(self.metadata[tag]) self.ogg.save() - def get_args(self,options=None): + def get_args(self, options=None): """Get process options and return arguments for the encoder""" args = [] if not options is None: @@ -133,13 +133,13 @@ def get_args(self,options=None): if not ('verbose' in self.options and self.options['verbose'] != '0'): args.append('-Q ') if 'ogg_bitrate' in self.options: - args.append('-b '+self.options['ogg_bitrate']) + args.append('-b ' + self.options['ogg_bitrate']) elif 'ogg_quality' in self.options: - args.append('-q '+self.options['ogg_quality']) + args.append('-q ' + self.options['ogg_quality']) else: - args.append('-b '+self.bitrate_default) + args.append('-b ' + self.bitrate_default) else: - args.append('-Q -b '+self.bitrate_default) + args.append('-Q -b ' + self.bitrate_default) for tag in self.metadata.keys(): value = clean_word(self.metadata[tag]) diff --git a/deefuzzer/tools/osc.py b/deefuzzer/tools/osc.py index 5e471a9..b4bb72a 100644 --- a/deefuzzer/tools/osc.py +++ b/deefuzzer/tools/osc.py @@ -40,10 +40,10 @@ class OSCController(Thread): - def __init__(self, port): Thread.__init__(self) import liblo + self.port = port try: self.server = liblo.Server(self.port) diff --git a/deefuzzer/tools/streamer.py b/deefuzzer/tools/streamer.py index 1d11091..bc95923 100644 --- a/deefuzzer/tools/streamer.py +++ b/deefuzzer/tools/streamer.py @@ -38,8 +38,8 @@ from threading import Thread -class HTTPStreamer(Thread): +class HTTPStreamer(Thread): protocol = 'http' host = str port = str @@ -58,6 +58,7 @@ class HTTPStreamer(Thread): def __init__(self): Thread.__init__(self) import pycurl + self.curl = pycurl.Curl() def set_callback(self, read_callback): @@ -68,8 +69,9 @@ def delay(self): def open(self): import pycurl - self.uri = self.protocol + '://' + self.host + ':' + str(self.port) + \ - self.mount + '?' + 'password=' + self.password + + self.uri = self.protocol + '://' + self.host + ':' + str(self.port) + self.uri += self.mount + '?' + 'password=' + self.password self.curl.setopt(pycurl.URL, self.uri) self.curl.setopt(pycurl.UPLOAD, 1) self.curl.setopt(pycurl.READFUNCTION, self.read_callback) diff --git a/deefuzzer/tools/twitt.py b/deefuzzer/tools/twitt.py index da26092..d487dd1 100644 --- a/deefuzzer/tools/twitt.py +++ b/deefuzzer/tools/twitt.py @@ -42,9 +42,9 @@ class Twitter: - def __init__(self, key, secret): import twitter + self.consumer_key = DEEFUZZER_CONSUMER_KEY self.consumer_secret = DEEFUZZER_CONSUMER_SECRET self.access_token_key = key diff --git a/deefuzzer/tools/utils.py b/deefuzzer/tools/utils.py index 43886cf..032d321 100644 --- a/deefuzzer/tools/utils.py +++ b/deefuzzer/tools/utils.py @@ -17,22 +17,24 @@ from itertools import chain from deefuzzer.tools import * -mimetypes.add_type('application/x-yaml','.yaml') +mimetypes.add_type('application/x-yaml', '.yaml') -def clean_word(word) : + +def clean_word(word): """ Return the word without excessive blank spaces, underscores and characters causing problem to exporters""" - word = re.sub("^[^\w]+","",word) #trim the beginning - word = re.sub("[^\w]+$","",word) #trim the end - word = re.sub("_+","_",word) #squeeze continuous _ to one _ - word = re.sub("^[^\w]+","",word) #trim the beginning _ - #word = string.replace(word,' ','_') - #word = string.capitalize(word) + word = re.sub("^[^\w]+", "", word) # trim the beginning + word = re.sub("[^\w]+$", "", word) # trim the end + word = re.sub("_+", "_", word) # squeeze continuous _ to one _ + word = re.sub("^[^\w]+", "", word) # trim the beginning _ + # word = string.replace(word,' ','_') + # word = string.capitalize(word) dict = '&[];"*:,' for letter in dict: - word = string.replace(word,letter,'_') + word = string.replace(word, letter, '_') return word + def get_file_info(media): file_name = media.split(os.sep)[-1] file_title = file_name.split('.')[:-1] @@ -40,9 +42,11 @@ def get_file_info(media): file_ext = file_name.split('.')[-1] return file_name, file_title, file_ext + def is_absolute_path(path): return os.sep == path[0] + def merge_defaults(setting, default): combined = {} for key in set(chain(setting, default)): @@ -58,6 +62,7 @@ def merge_defaults(setting, default): combined[key] = default[key] return combined + def replace_all(option, repl): if isinstance(option, list): r = [] @@ -72,37 +77,42 @@ def replace_all(option, repl): elif isinstance(option, str): r = option for key in repl.keys(): - r = r.replace('[' + key + ']', repl[key]) + r = r.replace('[' + key + ']', repl[key]) return r return option + def get_conf_dict(file): mime_type = mimetypes.guess_type(file)[0] # Do the type check first, so we don't load huge files that won't be used if 'xml' in mime_type: - confile = open(file,'r') + confile = open(file, 'r') data = confile.read() confile.close() - return xmltodict(data,'utf-8') + return xmltodict(data, 'utf-8') elif 'yaml' in mime_type: import yaml + def custom_str_constructor(loader, node): return loader.construct_scalar(node).encode('utf-8') + yaml.add_constructor(u'tag:yaml.org,2002:str', custom_str_constructor) - confile = open(file,'r') + confile = open(file, 'r') data = confile.read() confile.close() return yaml.load(data) elif 'json' in mime_type: import json - confile = open(file,'r') + + confile = open(file, 'r') data = confile.read() confile.close() return json.loads(data) return False + def folder_contains_music(folder): files = os.listdir(folder) for file in files: diff --git a/deefuzzer/tools/xml2yaml.py b/deefuzzer/tools/xml2yaml.py index ef78068..0e73d9d 100644 --- a/deefuzzer/tools/xml2yaml.py +++ b/deefuzzer/tools/xml2yaml.py @@ -1,7 +1,9 @@ - -import os, sys, yaml +import os +import sys +import yaml from deefuzzer.core import DeeFuzzer + path = sys.argv[-1] d = DeeFuzzer(path) name, ext = os.path.splitext(path) diff --git a/deefuzzer/tools/xmltodict.py b/deefuzzer/tools/xmltodict.py index 07bffcd..7d5e070 100644 --- a/deefuzzer/tools/xmltodict.py +++ b/deefuzzer/tools/xmltodict.py @@ -8,10 +8,11 @@ def haschilds(dom): # containing real tags opposed to just text. for childnode in dom.childNodes: if childnode.nodeName != "#text" and \ - childnode.nodeName != "#cdata-section": + childnode.nodeName != "#cdata-section": return True return False + def indexchilds(dom, enc): childsdict = dict() for childnode in dom.childNodes: @@ -32,6 +33,7 @@ def indexchilds(dom, enc): childsdict[name] = v return childsdict + def xmltodict(data, enc=None): dom = xml.dom.minidom.parseString(data.strip()) return indexchilds(dom, enc) From 73b238efd4d87d5b5dfb27263cf0dafbdb652aad Mon Sep 17 00:00:00 2001 From: achbed Date: Mon, 26 Jan 2015 16:35:35 -0600 Subject: [PATCH 61/82] Tabs to Spaces Signed-off-by: achbed --- deefuzzer/tools/xmltodict2.py | 706 +++++++++++++++++----------------- 1 file changed, 353 insertions(+), 353 deletions(-) diff --git a/deefuzzer/tools/xmltodict2.py b/deefuzzer/tools/xmltodict2.py index b669239..90b2ac8 100644 --- a/deefuzzer/tools/xmltodict2.py +++ b/deefuzzer/tools/xmltodict2.py @@ -2,7 +2,7 @@ """ xmltodict(): convert xml into tree of Python dicts. This was copied and modified from John Bair's recipe at aspn.activestate.com: - http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/149368 + http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/149368 """ import os import string @@ -16,12 +16,12 @@ #from dabo.lib.utils import resolvePath #app = dabo.dAppRef #if app is not None: - #default_encoding = app.Encoding + #default_encoding = app.Encoding #else: - #enc = locale.getlocale()[1] - #if enc is None: - #enc = dabo.defaultEncoding - #default_encoding = enc + #enc = locale.getlocale()[1] + #if enc is None: + #enc = dabo.defaultEncoding + #default_encoding = enc # Python seems to need to compile code with \n linesep: code_linesep = "\n" @@ -29,369 +29,369 @@ class Xml2Obj: - """XML to Object""" - def __init__(self): - self.root = None - self.nodeStack = [] - self.attsToSkip = [] - self._inCode = False - self._mthdName = "" - self._mthdCode = "" - self._codeDict = None - self._inProp = False - self._propName = "" - self._propData = "" - self._propDict = None - self._currPropAtt = "" - self._currPropDict = None - - - def StartElement(self, name, attributes): - """SAX start element even handler""" - if name == "code": - # This is code for the parent element - self._inCode = True - parent = self.nodeStack[-1] - if not parent.has_key("code"): - parent["code"] = {} - self._codeDict = parent["code"] - - elif name == "properties": - # These are the custom property definitions - self._inProp = True - self._propName = "" - self._propData = "" - parent = self.nodeStack[-1] - if not parent.has_key("properties"): - parent["properties"] = {} - self._propDict = parent["properties"] - - else: - if self._inCode: - self._mthdName = name.encode() - elif self._inProp: - if self._propName: - # In the middle of a prop definition - self._currPropAtt = name.encode() - else: - self._propName = name.encode() - self._currPropDict = {} - self._currPropAtt = "" - else: - element = {"name": name.encode()} - if len(attributes) > 0: - for att in self.attsToSkip: - if attributes.has_key(att): - del attributes[att] - element["attributes"] = attributes - - # Push element onto the stack and make it a child of parent - if len(self.nodeStack) > 0: - parent = self.nodeStack[-1] - if not parent.has_key("children"): - parent["children"] = [] - parent["children"].append(element) - else: - self.root = element - self.nodeStack.append(element) - - - def EndElement(self, name): - """SAX end element event handler""" - if self._inCode: - if name == "code": - self._inCode = False - self._codeDict = None - else: - # End of an individual method - mth = self._mthdCode.strip() - if not mth.endswith("\n"): - mth += "\n" - self._codeDict[self._mthdName] = mth - self._mthdName = "" - self._mthdCode = "" - elif self._inProp: - if name == "properties": - self._inProp = False - self._propDict = None - elif name == self._propName: - # End of an individual prop definition - self._propDict[self._propName] = self._currPropDict - self._propName = "" - else: - # end of a property attribute - self._currPropDict[self._currPropAtt] = self._propData - self._propData = self._currPropAtt = "" - else: - self.nodeStack = self.nodeStack[:-1] - - - def CharacterData(self, data): - """SAX character data event handler""" - if self._inCode or data.strip(): - data = data.replace("<", "<") - data = data.encode() - if self._inCode: - if self._mthdCode: - self._mthdCode += data - else: - self._mthdCode = data - elif self._inProp: - self._propData += data - else: - element = self.nodeStack[-1] - if not element.has_key("cdata"): - element["cdata"] = "" - element["cdata"] += data - - - def Parse(self, xml): - # Create a SAX parser - Parser = expat.ParserCreate() - # SAX event handlers - Parser.StartElementHandler = self.StartElement - Parser.EndElementHandler = self.EndElement - Parser.CharacterDataHandler = self.CharacterData - # Parse the XML File - ParserStatus = Parser.Parse(xml, 1) - return self.root - - - def ParseFromFile(self, filename): - return self.Parse(open(filename,"r").read()) + """XML to Object""" + def __init__(self): + self.root = None + self.nodeStack = [] + self.attsToSkip = [] + self._inCode = False + self._mthdName = "" + self._mthdCode = "" + self._codeDict = None + self._inProp = False + self._propName = "" + self._propData = "" + self._propDict = None + self._currPropAtt = "" + self._currPropDict = None + + + def StartElement(self, name, attributes): + """SAX start element even handler""" + if name == "code": + # This is code for the parent element + self._inCode = True + parent = self.nodeStack[-1] + if not parent.has_key("code"): + parent["code"] = {} + self._codeDict = parent["code"] + + elif name == "properties": + # These are the custom property definitions + self._inProp = True + self._propName = "" + self._propData = "" + parent = self.nodeStack[-1] + if not parent.has_key("properties"): + parent["properties"] = {} + self._propDict = parent["properties"] + + else: + if self._inCode: + self._mthdName = name.encode() + elif self._inProp: + if self._propName: + # In the middle of a prop definition + self._currPropAtt = name.encode() + else: + self._propName = name.encode() + self._currPropDict = {} + self._currPropAtt = "" + else: + element = {"name": name.encode()} + if len(attributes) > 0: + for att in self.attsToSkip: + if attributes.has_key(att): + del attributes[att] + element["attributes"] = attributes + + # Push element onto the stack and make it a child of parent + if len(self.nodeStack) > 0: + parent = self.nodeStack[-1] + if not parent.has_key("children"): + parent["children"] = [] + parent["children"].append(element) + else: + self.root = element + self.nodeStack.append(element) + + + def EndElement(self, name): + """SAX end element event handler""" + if self._inCode: + if name == "code": + self._inCode = False + self._codeDict = None + else: + # End of an individual method + mth = self._mthdCode.strip() + if not mth.endswith("\n"): + mth += "\n" + self._codeDict[self._mthdName] = mth + self._mthdName = "" + self._mthdCode = "" + elif self._inProp: + if name == "properties": + self._inProp = False + self._propDict = None + elif name == self._propName: + # End of an individual prop definition + self._propDict[self._propName] = self._currPropDict + self._propName = "" + else: + # end of a property attribute + self._currPropDict[self._currPropAtt] = self._propData + self._propData = self._currPropAtt = "" + else: + self.nodeStack = self.nodeStack[:-1] + + + def CharacterData(self, data): + """SAX character data event handler""" + if self._inCode or data.strip(): + data = data.replace("<", "<") + data = data.encode() + if self._inCode: + if self._mthdCode: + self._mthdCode += data + else: + self._mthdCode = data + elif self._inProp: + self._propData += data + else: + element = self.nodeStack[-1] + if not element.has_key("cdata"): + element["cdata"] = "" + element["cdata"] += data + + + def Parse(self, xml): + # Create a SAX parser + Parser = expat.ParserCreate() + # SAX event handlers + Parser.StartElementHandler = self.StartElement + Parser.EndElementHandler = self.EndElement + Parser.CharacterDataHandler = self.CharacterData + # Parse the XML File + ParserStatus = Parser.Parse(xml, 1) + return self.root + + + def ParseFromFile(self, filename): + return self.Parse(open(filename,"r").read()) def xmltodict(xml, attsToSkip=[], addCodeFile=False): - """Given an xml string or file, return a Python dictionary.""" - parser = Xml2Obj() - parser.attsToSkip = attsToSkip - isPath = os.path.exists(xml) - errmsg = "" - if eol not in xml and isPath: - # argument was a file - try: - ret = parser.ParseFromFile(xml) - except expat.ExpatError, e: - errmsg = _("The XML in '%s' is not well-formed and cannot be parsed: %s") % (xml, e) - else: - # argument must have been raw xml: - if not xml.strip().startswith(" 127: - chars.append("&#%s;" % ord(char)) - else: - chars.append(char) - val = "".join(chars) - val = val.replace("<", "<").replace(">", ">") - return "%s%s%s" % (qt, val, qt) + """Add surrounding quotes to the string, and escape + any illegal XML characters. + """ + if not isinstance(val, basestring): + val = str(val) + if not isinstance(val, unicode): + val = unicode(val, default_encoding) + if noQuote: + qt = '' + else: + qt = '"' + slsh = "\\" +# val = val.replace(slsh, slsh+slsh) + if not noEscape: + # First escape internal ampersands. We need to double them up due to a + # quirk in wxPython and the way it displays this character. + val = val.replace("&", "&&") + # Escape any internal quotes + val = val.replace('"', '"').replace("'", "'") + # Escape any high-order characters + chars = [] + for pos, char in enumerate(list(val)): + if ord(char) > 127: + chars.append("&#%s;" % ord(char)) + else: + chars.append(char) + val = "".join(chars) + val = val.replace("<", "<").replace(">", ">") + return "%s%s%s" % (qt, val, qt) def dicttoxml(dct, level=0, header=None, linesep=None): - """Given a Python dictionary, return an xml string. - - The dictionary must be in the format returned by dicttoxml(), with keys - on "attributes", "code", "cdata", "name", and "children". - - Send your own XML header, otherwise a default one will be used. - - The linesep argument is a dictionary, with keys on levels, allowing the - developer to add extra whitespace depending on the level. - """ - att = "" - ret = "" - - if dct.has_key("attributes"): - for key, val in dct["attributes"].items(): - # Some keys are already handled. - noEscape = key in ("sizerInfo",) - val = escQuote(val, noEscape) - att += " %s=%s" % (key, val) - ret += "%s<%s%s" % ("\t" * level, dct["name"], att) - - if (not dct.has_key("cdata") and not dct.has_key("children") - and not dct.has_key("code") and not dct.has_key("properties")): - ret += " />%s" % eol - else: - ret += ">" - if dct.has_key("cdata"): - ret += "%s" % dct["cdata"].replace("<", "<") - - if dct.has_key("code"): - if len(dct["code"].keys()): - ret += "%s%s%s" % (eol, "\t" * (level+1), eol) - methodTab = "\t" * (level+2) - for mthd, cd in dct["code"].items(): - # Convert \n's in the code to eol: - cd = eol.join(cd.splitlines()) - - # Make sure that the code ends with a linefeed - if not cd.endswith(eol): - cd += eol - - ret += "%s<%s>%s%s%s" % (methodTab, - mthd, eol, cd, eol, - methodTab, mthd, eol) - ret += "%s%s" % ("\t" * (level+1), eol) - - if dct.has_key("properties"): - if len(dct["properties"].keys()): - ret += "%s%s%s" % (eol, "\t" * (level+1), eol) - currTab = "\t" * (level+2) - for prop, val in dct["properties"].items(): - ret += "%s<%s>%s" % (currTab, prop, eol) - for propItm, itmVal in val.items(): - itmTab = "\t" * (level+3) - ret += "%s<%s>%s%s" % (itmTab, propItm, itmVal, - propItm, eol) - ret += "%s%s" % (currTab, prop, eol) - ret += "%s%s" % ("\t" * (level+1), eol) - - if dct.has_key("children") and len(dct["children"]) > 0: - ret += eol - for child in dct["children"]: - ret += dicttoxml(child, level+1, linesep=linesep) - indnt = "" - if ret.endswith(eol): - # Indent the closing tag - indnt = ("\t" * level) - ret += "%s%s" % (indnt, dct["name"], eol) - - if linesep: - ret += linesep.get(level, "") - - if level == 0: - if header is None: - header = '%s' \ - % (default_encoding, eol) - ret = header + ret - - return ret + """Given a Python dictionary, return an xml string. + + The dictionary must be in the format returned by dicttoxml(), with keys + on "attributes", "code", "cdata", "name", and "children". + + Send your own XML header, otherwise a default one will be used. + + The linesep argument is a dictionary, with keys on levels, allowing the + developer to add extra whitespace depending on the level. + """ + att = "" + ret = "" + + if dct.has_key("attributes"): + for key, val in dct["attributes"].items(): + # Some keys are already handled. + noEscape = key in ("sizerInfo",) + val = escQuote(val, noEscape) + att += " %s=%s" % (key, val) + ret += "%s<%s%s" % ("\t" * level, dct["name"], att) + + if (not dct.has_key("cdata") and not dct.has_key("children") + and not dct.has_key("code") and not dct.has_key("properties")): + ret += " />%s" % eol + else: + ret += ">" + if dct.has_key("cdata"): + ret += "%s" % dct["cdata"].replace("<", "<") + + if dct.has_key("code"): + if len(dct["code"].keys()): + ret += "%s%s%s" % (eol, "\t" * (level+1), eol) + methodTab = "\t" * (level+2) + for mthd, cd in dct["code"].items(): + # Convert \n's in the code to eol: + cd = eol.join(cd.splitlines()) + + # Make sure that the code ends with a linefeed + if not cd.endswith(eol): + cd += eol + + ret += "%s<%s>%s%s%s" % (methodTab, + mthd, eol, cd, eol, + methodTab, mthd, eol) + ret += "%s%s" % ("\t" * (level+1), eol) + + if dct.has_key("properties"): + if len(dct["properties"].keys()): + ret += "%s%s%s" % (eol, "\t" * (level+1), eol) + currTab = "\t" * (level+2) + for prop, val in dct["properties"].items(): + ret += "%s<%s>%s" % (currTab, prop, eol) + for propItm, itmVal in val.items(): + itmTab = "\t" * (level+3) + ret += "%s<%s>%s%s" % (itmTab, propItm, itmVal, + propItm, eol) + ret += "%s%s" % (currTab, prop, eol) + ret += "%s%s" % ("\t" * (level+1), eol) + + if dct.has_key("children") and len(dct["children"]) > 0: + ret += eol + for child in dct["children"]: + ret += dicttoxml(child, level+1, linesep=linesep) + indnt = "" + if ret.endswith(eol): + # Indent the closing tag + indnt = ("\t" * level) + ret += "%s%s" % (indnt, dct["name"], eol) + + if linesep: + ret += linesep.get(level, "") + + if level == 0: + if header is None: + header = '%s' \ + % (default_encoding, eol) + ret = header + ret + + return ret def flattenClassDict(cd, retDict=None): - """Given a dict containing a series of nested objects such as would - be created by restoring from a cdxml file, returns a dict with all classIDs - as keys, and a dict as the corresponding value. The dict value will have - keys for the attributes and/or code, depending on what was in the original - dict. The end result is to take a nested dict structure and return a flattened - dict with all objects at the top level. - """ - if retDict is None: - retDict = {} - atts = cd.get("attributes", {}) - props = cd.get("properties", {}) - kids = cd.get("children", []) - code = cd.get("code", {}) - classID = atts.get("classID", "") - classFile = resolvePath(atts.get("designerClass", "")) - superclass = resolvePath(atts.get("superclass", "")) - superclassID = atts.get("superclassID", "") - if superclassID and os.path.exists(superclass): - # Get the superclass info - superCD = xmltodict(superclass, addCodeFile=True) - flattenClassDict(superCD, retDict) - if classID: - if os.path.exists(classFile): - # Get the class info - classCD = xmltodict(classFile, addCodeFile=True) - classAtts = classCD.get("attributes", {}) - classProps = classCD.get("properties", {}) - classCode = classCD.get("code", {}) - classKids = classCD.get("children", []) - currDict = retDict.get(classID, {}) - retDict[classID] = {"attributes": classAtts, "code": classCode, - "properties": classProps} - retDict[classID].update(currDict) - # Now update the child objects in the dict - for kid in classKids: - flattenClassDict(kid, retDict) - else: - # Not a file; most likely just a component in another class - currDict = retDict.get(classID, {}) - retDict[classID] = {"attributes": atts, "code": code, - "properties": props} - retDict[classID].update(currDict) - if kids: - for kid in kids: - flattenClassDict(kid, retDict) - return retDict + """Given a dict containing a series of nested objects such as would + be created by restoring from a cdxml file, returns a dict with all classIDs + as keys, and a dict as the corresponding value. The dict value will have + keys for the attributes and/or code, depending on what was in the original + dict. The end result is to take a nested dict structure and return a flattened + dict with all objects at the top level. + """ + if retDict is None: + retDict = {} + atts = cd.get("attributes", {}) + props = cd.get("properties", {}) + kids = cd.get("children", []) + code = cd.get("code", {}) + classID = atts.get("classID", "") + classFile = resolvePath(atts.get("designerClass", "")) + superclass = resolvePath(atts.get("superclass", "")) + superclassID = atts.get("superclassID", "") + if superclassID and os.path.exists(superclass): + # Get the superclass info + superCD = xmltodict(superclass, addCodeFile=True) + flattenClassDict(superCD, retDict) + if classID: + if os.path.exists(classFile): + # Get the class info + classCD = xmltodict(classFile, addCodeFile=True) + classAtts = classCD.get("attributes", {}) + classProps = classCD.get("properties", {}) + classCode = classCD.get("code", {}) + classKids = classCD.get("children", []) + currDict = retDict.get(classID, {}) + retDict[classID] = {"attributes": classAtts, "code": classCode, + "properties": classProps} + retDict[classID].update(currDict) + # Now update the child objects in the dict + for kid in classKids: + flattenClassDict(kid, retDict) + else: + # Not a file; most likely just a component in another class + currDict = retDict.get(classID, {}) + retDict[classID] = {"attributes": atts, "code": code, + "properties": props} + retDict[classID].update(currDict) + if kids: + for kid in kids: + flattenClassDict(kid, retDict) + return retDict def addInheritedInfo(src, super, updateCode=False): - """Called recursively on the class container structure, modifying - the attributes to incorporate superclass information. When the - 'updateCode' parameter is True, superclass code is added to the - object's code - """ - atts = src.get("attributes", {}) - props = src.get("properties", {}) - kids = src.get("children", []) - code = src.get("code", {}) - classID = atts.get("classID", "") - if classID: - superInfo = super.get(classID, {"attributes": {}, "code": {}, "properties": {}}) - src["attributes"] = superInfo["attributes"].copy() - src["attributes"].update(atts) - src["properties"] = superInfo.get("properties", {}).copy() - src["properties"].update(props) - if updateCode: - src["code"] = superInfo["code"].copy() - src["code"].update(code) - if kids: - for kid in kids: - addInheritedInfo(kid, super, updateCode) + """Called recursively on the class container structure, modifying + the attributes to incorporate superclass information. When the + 'updateCode' parameter is True, superclass code is added to the + object's code + """ + atts = src.get("attributes", {}) + props = src.get("properties", {}) + kids = src.get("children", []) + code = src.get("code", {}) + classID = atts.get("classID", "") + if classID: + superInfo = super.get(classID, {"attributes": {}, "code": {}, "properties": {}}) + src["attributes"] = superInfo["attributes"].copy() + src["attributes"].update(atts) + src["properties"] = superInfo.get("properties", {}).copy() + src["properties"].update(props) + if updateCode: + src["code"] = superInfo["code"].copy() + src["code"].update(code) + if kids: + for kid in kids: + addInheritedInfo(kid, super, updateCode) #if __name__ == "__main__": - #test_dict = {"name": "test", "attributes":{"path": "c:\\temp\\name", - #"problemChars": "Welcome to \xc2\xae".decode("latin-1")}} - #print "test_dict:", test_dict - #xml = dicttoxml(test_dict) - #print "xml:", xml - #test_dict2 = xmltodict(xml) - #print "test_dict2:", test_dict2 - #print "same?:", test_dict == test_dict2 + #test_dict = {"name": "test", "attributes":{"path": "c:\\temp\\name", + #"problemChars": "Welcome to \xc2\xae".decode("latin-1")}} + #print "test_dict:", test_dict + #xml = dicttoxml(test_dict) + #print "xml:", xml + #test_dict2 = xmltodict(xml) + #print "test_dict2:", test_dict2 + #print "same?:", test_dict == test_dict2 From edb4b03659f86d41f1e69cfc8243591257f2121a Mon Sep 17 00:00:00 2001 From: achbed Date: Mon, 26 Jan 2015 16:44:26 -0600 Subject: [PATCH 62/82] Additional coding style cleanup Signed-off-by: achbed --- deefuzzer/core.py | 27 ++++++++++--------- deefuzzer/station.py | 1 - deefuzzer/tools/PyRSS2Gen.py | 1 + deefuzzer/tools/ogg.py | 2 +- deefuzzer/tools/xmltodict2.py | 49 ++++++++++++++++------------------- 5 files changed, 38 insertions(+), 42 deletions(-) diff --git a/deefuzzer/core.py b/deefuzzer/core.py index 7fbaf07..ee82fd5 100644 --- a/deefuzzer/core.py +++ b/deefuzzer/core.py @@ -46,7 +46,7 @@ from deefuzzer.station import * from deefuzzer.tools import * -mimetypes.add_type('application/x-yaml','.yaml') +mimetypes.add_type('application/x-yaml', '.yaml') class DeeFuzzer(Thread): @@ -115,7 +115,6 @@ def __init__(self, conf_file): self._info('Using libshout version %s' % shout.version()) self._info('Number of stations : ' + str(len(self.station_settings))) - def _log(self, level, msg): try: obj = {'msg': 'Core: ' + str(msg), 'level': level} @@ -137,7 +136,7 @@ def set_m3u_playlist(self): m3u.write('#EXTM3U\n') for k in self.station_instances.keys(): s = self.station_instances[k] - m3u.write('#EXTINF:%s,%s - %s\n' % ('-1',s.short_name, s.channel.name)) + m3u.write('#EXTINF:%s,%s - %s\n' % ('-1', s.short_name, s.channel.name)) m3u.write('http://' + s.channel.host + ':' + str(s.channel.port) + s.channel.mount + '\n') m3u.close() self._info('Writing M3U file to : ' + self.m3u) @@ -146,12 +145,12 @@ def create_stations_fromfolder(self): """Scan a folder for subfolders containing media, and make stations from them all.""" options = self.watchfolder - if not 'folder' in options.keys(): + if 'folder' not in options.keys(): # We have no folder specified. Bail. return if self.mainLoop: - if not 'livecreation' in options.keys(): + if 'livecreation' not in options.keys(): # We have no folder specified. Bail. return @@ -167,9 +166,9 @@ def create_stations_fromfolder(self): # This makes the log file a lot more verbose. Commented out since we report on new stations anyway. # self._info('Scanning folder ' + folder + ' for stations') - if not 'infos' in options.keys(): + if 'infos' not in options.keys(): options['infos'] = {} - if not 'short_name' in options['infos'].keys(): + if 'short_name' not in options['infos'].keys(): options['infos']['short_name'] = '[name]' files = os.listdir(folder) @@ -182,9 +181,9 @@ def create_stations_fromfolder(self): def station_exists(self, name): try: for s in self.station_settings: - if not 'infos' in s.keys(): + if 'infos' not in s.keys(): continue - if not 'short_name' in s['infos'].keys(): + if 'short_name' not in s['infos'].keys(): continue if s['infos']['short_name'] == name: return True @@ -201,11 +200,11 @@ def create_station(self, folder, options): if self.station_exists(name): return self._info('Creating station for folder ' + folder) - d = dict(path=folder,name=name) + d = dict(path=folder, name=name) for i in options.keys(): - if not 'folder' in i: + if 'folder' not in i: s[i] = replace_all(options[i], d) - if not 'media' in s.keys(): + if 'media' not in s.keys(): s['media'] = {} s['media']['source'] = folder @@ -276,7 +275,7 @@ def run(self): if 'station_name' in self.station_settings[i].keys(): name = self.station_settings[i]['station_name'] - if not 'retries' in self.station_settings[i].keys(): + if 'retries' not in self.station_settings[i].keys(): self.station_settings[i]['retries'] = 0 try: @@ -289,7 +288,7 @@ def run(self): if self.maxretry >= 0 and self.station_settings[i]['retries'] <= self.maxretry: # Station passed the max retries count is will not be reloaded - if not 'station_stop_logged' in self.station_settings[i].keys(): + if 'station_stop_logged' not in self.station_settings[i].keys(): self._err('Station ' + name + ' is stopped and will not be restarted.') self.station_settings[i]['station_stop_logged'] = True continue diff --git a/deefuzzer/station.py b/deefuzzer/station.py index 36124dd..59eeb15 100644 --- a/deefuzzer/station.py +++ b/deefuzzer/station.py @@ -806,7 +806,6 @@ def __twitter_should_update(self): return False return True - def icecastloop_metadata(self): try: self.update_twitter_current() diff --git a/deefuzzer/tools/PyRSS2Gen.py b/deefuzzer/tools/PyRSS2Gen.py index a2122f7..eb4edc3 100644 --- a/deefuzzer/tools/PyRSS2Gen.py +++ b/deefuzzer/tools/PyRSS2Gen.py @@ -8,6 +8,7 @@ import datetime + # Could make this the base class; will need to add 'publish' class WriteXmlMixin: def __init__(self): diff --git a/deefuzzer/tools/ogg.py b/deefuzzer/tools/ogg.py index 3c196a1..bca591d 100644 --- a/deefuzzer/tools/ogg.py +++ b/deefuzzer/tools/ogg.py @@ -128,7 +128,7 @@ def write_tags(self): def get_args(self, options=None): """Get process options and return arguments for the encoder""" args = [] - if not options is None: + if options is not None: self.options = options if not ('verbose' in self.options and self.options['verbose'] != '0'): args.append('-Q ') diff --git a/deefuzzer/tools/xmltodict2.py b/deefuzzer/tools/xmltodict2.py index 90b2ac8..1724ec3 100644 --- a/deefuzzer/tools/xmltodict2.py +++ b/deefuzzer/tools/xmltodict2.py @@ -10,18 +10,18 @@ from xml.parsers import expat # If we're in Dabo, get the default encoding. -#import dabo -#import dabo.lib.DesignerUtils as desUtil -#from dabo.dLocalize import _ -#from dabo.lib.utils import resolvePath -#app = dabo.dAppRef -#if app is not None: - #default_encoding = app.Encoding -#else: - #enc = locale.getlocale()[1] - #if enc is None: - #enc = dabo.defaultEncoding - #default_encoding = enc +# import dabo +# import dabo.lib.DesignerUtils as desUtil +# from dabo.dLocalize import _ +# from dabo.lib.utils import resolvePath +# app = dabo.dAppRef +# if app is not None: + # default_encoding = app.Encoding +# else: + # enc = locale.getlocale()[1] + # if enc is None: + # enc = dabo.defaultEncoding + # default_encoding = enc # Python seems to need to compile code with \n linesep: code_linesep = "\n" @@ -45,7 +45,6 @@ def __init__(self): self._currPropAtt = "" self._currPropDict = None - def StartElement(self, name, attributes): """SAX start element even handler""" if name == "code": @@ -95,7 +94,6 @@ def StartElement(self, name, attributes): self.root = element self.nodeStack.append(element) - def EndElement(self, name): """SAX end element event handler""" if self._inCode: @@ -156,7 +154,6 @@ def Parse(self, xml): ParserStatus = Parser.Parse(xml, 1) return self.root - def ParseFromFile(self, filename): return self.Parse(open(filename,"r").read()) @@ -274,7 +271,7 @@ def dicttoxml(dct, level=0, header=None, linesep=None): ret += "%s<%s>%s%s%s" % (methodTab, mthd, eol, cd, eol, methodTab, mthd, eol) - ret += "%s%s" % ("\t" * (level+1), eol) + ret += "%s%s" % ("\t" * (level+1), eol) if dct.has_key("properties"): if len(dct["properties"].keys()): @@ -287,7 +284,7 @@ def dicttoxml(dct, level=0, header=None, linesep=None): ret += "%s<%s>%s%s" % (itmTab, propItm, itmVal, propItm, eol) ret += "%s%s" % (currTab, prop, eol) - ret += "%s%s" % ("\t" * (level+1), eol) + ret += "%s%s" % ("\t" * (level+1), eol) if dct.has_key("children") and len(dct["children"]) > 0: ret += eol @@ -386,12 +383,12 @@ def addInheritedInfo(src, super, updateCode=False): -#if __name__ == "__main__": - #test_dict = {"name": "test", "attributes":{"path": "c:\\temp\\name", - #"problemChars": "Welcome to \xc2\xae".decode("latin-1")}} - #print "test_dict:", test_dict - #xml = dicttoxml(test_dict) - #print "xml:", xml - #test_dict2 = xmltodict(xml) - #print "test_dict2:", test_dict2 - #print "same?:", test_dict == test_dict2 +# if __name__ == "__main__": + # test_dict = {"name": "test", "attributes":{"path": "c:\\temp\\name", + # "problemChars": "Welcome to \xc2\xae".decode("latin-1")}} + # print "test_dict:", test_dict + # xml = dicttoxml(test_dict) + # print "xml:", xml + # test_dict2 = xmltodict(xml) + # print "test_dict2:", test_dict2 + # print "same?:", test_dict == test_dict2 From d813dd8d18aa12c406ca0c234596cdd7ec84fb71 Mon Sep 17 00:00:00 2001 From: achbed Date: Mon, 26 Jan 2015 17:41:42 -0600 Subject: [PATCH 63/82] Lots of style fixes Also: Fixes for a few "using unset variable" cases Began removing calls to .has_key() (depreciated) Fixed reference issue on raiseErrors flag (should have been self.raiseErrors) Signed-off-by: achbed --- deefuzzer/core.py | 18 ++++--- deefuzzer/station.py | 37 +++++++------- deefuzzer/streamer.py | 4 +- deefuzzer/tools/PyRSS2Gen.py | 87 +++++++++++++++++--------------- deefuzzer/tools/mp3.py | 36 +++++++------ deefuzzer/tools/ogg.py | 15 +++--- deefuzzer/tools/xmltodict.py | 3 +- deefuzzer/tools/xmltodict2.py | 65 ++++++++++++++---------- scripts/osc/osc_jingles_start.py | 3 +- scripts/osc/osc_jingles_stop.py | 3 +- scripts/osc/osc_player_fast.py | 3 +- scripts/osc/osc_player_next.py | 3 +- scripts/osc/osc_player_slow.py | 3 +- scripts/osc/osc_record_start.py | 3 +- scripts/osc/osc_record_stop.py | 3 +- scripts/osc/osc_relay_start.py | 3 +- scripts/osc/osc_relay_stop.py | 3 +- scripts/osc/osc_twitter_start.py | 3 +- scripts/osc/osc_twitter_stop.py | 3 +- setup.py | 47 ++++++++--------- 20 files changed, 191 insertions(+), 154 deletions(-) diff --git a/deefuzzer/core.py b/deefuzzer/core.py index ee82fd5..954c290 100644 --- a/deefuzzer/core.py +++ b/deefuzzer/core.py @@ -68,7 +68,7 @@ def __init__(self, conf_file): self.conf_file = conf_file self.conf = get_conf_dict(self.conf_file) - if not 'deefuzzer' in self.conf.keys(): + if 'deefuzzer' not in self.conf.keys(): return # Get the log setting first (if possible) @@ -266,12 +266,12 @@ def run(self): while True: self.create_stations_fromfolder() ns_new = len(self.station_settings) - if(ns_new > ns): + if ns_new > ns: self._info('Loading new stations') for i in range(0, ns_new): + name = '' try: - name = '' if 'station_name' in self.station_settings[i].keys(): name = self.station_settings[i]['station_name'] @@ -294,17 +294,21 @@ def run(self): continue self.station_settings[i]['retries'] += 1 - self._info('Restarting station ' + name + ' (try ' + str(self.station_settings[i]['retries']) + ')') + trynum = str(self.station_settings[i]['retries']) + self._info('Restarting station ' + name + ' (try ' + trynum + ')') except Exception as e: self._err('Error checking status for ' + name) self._err(str(e)) - if not ignoreErrors: + if not self.ignoreErrors: raise # Apply station defaults if they exist if 'stationdefaults' in self.conf['deefuzzer']: if isinstance(self.conf['deefuzzer']['stationdefaults'], dict): - self.station_settings[i] = merge_defaults(self.station_settings[i], self.conf['deefuzzer']['stationdefaults']) + self.station_settings[i] = merge_defaults( + self.station_settings[i], + self.conf['deefuzzer']['stationdefaults'] + ) if name == '': name = 'Station ' + str(i) @@ -329,7 +333,7 @@ def run(self): self._err('Error validating station ' + name) except Exception: self._err('Error initializing station ' + name) - if not ignoreErrors: + if not self.ignoreErrors: raise continue diff --git a/deefuzzer/station.py b/deefuzzer/station.py index 59eeb15..4e34f09 100644 --- a/deefuzzer/station.py +++ b/deefuzzer/station.py @@ -375,16 +375,18 @@ def record_callback(self, path, value): if self.type == 'icecast': date = datetime.datetime.now().strftime("%Y") + media = None if self.channel.format == 'mp3': media = Mp3(self.record_dir + os.sep + self.rec_file) if self.channel.format == 'ogg': media = Ogg(self.record_dir + os.sep + self.rec_file) - media.metadata = {'artist': self.artist.encode('utf-8'), - 'title': self.title.encode('utf-8'), - 'album': self.short_name.encode('utf-8'), - 'genre': self.channel.genre.encode('utf-8'), - 'date': date.encode('utf-8'), } - media.write_tags() + if media: + media.metadata = {'artist': self.artist.encode('utf-8'), + 'title': self.title.encode('utf-8'), + 'album': self.short_name.encode('utf-8'), + 'genre': self.channel.genre.encode('utf-8'), + 'date': date.encode('utf-8'), } + media.write_tags() self.record_mode = value message = "received OSC message '%s' with arguments '%d'" % (path, value) @@ -452,23 +454,24 @@ def tweet(self): for media_obj in new_tracks_objs: title = '' artist = '' - if media_obj.metadata.has_key('title'): + if 'title' in media_obj.metadata: title = media_obj.metadata['title'] - if media_obj.metadata.has_key('artist'): + if 'artist' in media_obj.metadata: artist = media_obj.metadata['artist'] if not (title or artist): song = str(media_obj.file_name) else: song = artist + ' - ' + title - song = song.encode('utf-8') - artist = artist.encode('utf-8') - artist_names = artist.split(' ') - artist_tags = ' #'.join(list(set(artist_names) - {'&', '-'})) - message = '#NEWTRACK ! %s #%s on #%s RSS: ' % \ - (song.replace('_', ' '), artist_tags, self.short_name) - message = message[:113] + self.feeds_url - self.update_twitter(message) + song = song.encode('utf-8') + artist = artist.encode('utf-8') + + artist_names = artist.split(' ') + artist_tags = ' #'.join(list(set(artist_names) - {'&', '-'})) + message = '#NEWTRACK ! %s #%s on #%s RSS: ' % \ + (song.replace('_', ' '), artist_tags, self.short_name) + message = message[:113] + self.feeds_url + self.update_twitter(message) def get_next_media(self): # Init playlist @@ -728,7 +731,7 @@ def update_twitter_current(self): if not self.__twitter_should_update(): return artist_names = self.artist.split(' ') - artist_tags = ' #'.join(list(set(artist_names) - {'&', '-'})) + # artist_tags = ' #'.join(list(set(artist_names) - {'&', '-'})) message = '%s %s on #%s' % (self.prefix, self.song, self.short_name) tags = '#' + ' #'.join(self.twitter_tags) message = message + ' ' + tags diff --git a/deefuzzer/streamer.py b/deefuzzer/streamer.py index 49d9909..54fb5ca 100644 --- a/deefuzzer/streamer.py +++ b/deefuzzer/streamer.py @@ -53,7 +53,7 @@ class HTTPStreamer(Thread): decription = str format = str url = str - delay = 0 + _delay = 0 def __init__(self): Thread.__init__(self) @@ -65,7 +65,7 @@ def set_callback(self, read_callback): self.read_callback = read_callback def delay(self): - return self.delay + return self._delay def open(self): import pycurl diff --git a/deefuzzer/tools/PyRSS2Gen.py b/deefuzzer/tools/PyRSS2Gen.py index eb4edc3..56b249a 100644 --- a/deefuzzer/tools/PyRSS2Gen.py +++ b/deefuzzer/tools/PyRSS2Gen.py @@ -32,7 +32,9 @@ def to_xml(self, encoding="iso-8859-1"): return f.getvalue() -def _element(handler, name, obj, d={}): +def _element(handler, name, obj, d=None): + if not d: + d = {} if isinstance(obj, basestring) or obj is None: # special-case handling to make the API easier # to use for the common case. @@ -232,11 +234,11 @@ def __init__(self, url, length, type): self.type = type def publish(self, handler): - _element(handler, "enclosure", None, - {"url": self.url, - "length": str(self.length), - "type": self.type, - }) + _element(handler, "enclosure", None, { + "url": self.url, + "length": str(self.length), + "type": self.type + }) class Source: @@ -296,31 +298,31 @@ class RSS2(WriteXmlMixin): rss_attrs = {"version": "2.0"} element_attrs = {} - def __init__(self, - title, - link, - description, - - language=None, - copyright=None, - managingEditor=None, - webMaster=None, - pubDate=None, # a datetime, *in* *GMT* - lastBuildDate=None, # a datetime - - categories=None, # list of strings or Category - generator=_generator_name, - docs="http://blogs.law.harvard.edu/tech/rss", - cloud=None, # a Cloud - ttl=None, # integer number of minutes - - image=None, # an Image - rating=None, # a string; I don't know how it's used - textInput=None, # a TextInput - skipHours=None, # a SkipHours with a list of integers - skipDays=None, # a SkipDays with a list of strings - - items=None, # list of RSSItems + def __init__( + self, + title, + link, + description, + + language=None, + copyright=None, + managingEditor=None, + webMaster=None, + pubDate=None, # a datetime, *in* *GMT* + lastBuildDate=None, # a datetime + + categories=None, # list of strings or Category + generator=_generator_name, + docs="http://blogs.law.harvard.edu/tech/rss", + cloud=None, # a Cloud + ttl=None, # integer number of minutes + + image=None, # an Image + rating=None, # a string; I don't know how it's used + textInput=None, # a TextInput + skipHours=None, # a SkipHours with a list of integers + skipDays=None, # a SkipDays with a list of strings + items=None # list of RSSItems ): self.title = title self.link = link @@ -417,17 +419,18 @@ class RSSItem(WriteXmlMixin): """Publish an RSS Item""" element_attrs = {} - def __init__(self, - title=None, # string - link=None, # url as string - description=None, # string - author=None, # email address as string - categories=None, # list of string or Category - comments=None, # url as string - enclosure=None, # an Enclosure - guid=None, # a unique string - pubDate=None, # a datetime - source=None, # a Source + def __init__( + self, + title=None, # string + link=None, # url as string + description=None, # string + author=None, # email address as string + categories=None, # list of string or Category + comments=None, # url as string + enclosure=None, # an Enclosure + guid=None, # a unique string + pubDate=None, # a datetime + source=None # a Source ): if title is None and description is None: diff --git a/deefuzzer/tools/mp3.py b/deefuzzer/tools/mp3.py index fac15a1..eb3f61e 100644 --- a/deefuzzer/tools/mp3.py +++ b/deefuzzer/tools/mp3.py @@ -60,14 +60,15 @@ def __init__(self, media): self.options = {} self.bitrate_default = '192' self.cache_dir = os.sep + 'tmp' - self.keys2id3 = {'title': 'TIT2', - 'artist': 'TPE1', - 'album': 'TALB', - 'date': 'TDRC', - 'comment': 'COMM', - 'country': 'COUNTRY', - 'genre': 'TCON', - 'copyright': 'TCOP', + self.keys2id3 = { + 'title': 'TIT2', + 'artist': 'TPE1', + 'album': 'TALB', + 'date': 'TDRC', + 'comment': 'COMM', + 'country': 'COUNTRY', + 'genre': 'TCON', + 'copyright': 'TCOP' } self.mp3 = MP3(self.media, ID3=EasyID3) self.info = self.mp3.info @@ -76,14 +77,15 @@ def __init__(self, media): try: self.metadata = self.get_file_metadata() except: - self.metadata = {'title': '', - 'artist': '', - 'album': '', - 'date': '', - 'comment': '', - 'country': '', - 'genre': '', - 'copyright': '', + self.metadata = { + 'title': '', + 'artist': '', + 'album': '', + 'date': '', + 'comment': '', + 'country': '', + 'genre': '', + 'copyright': '' } self.description = self.get_description() @@ -128,6 +130,7 @@ def write_tags(self): self.mp3.tags['TIT2'] = id3.TIT2(encoding=2, text=u'text') self.mp3.save() + ''' # media_id3 = id3.ID3(self.media) # for tag in self.metadata.keys(): # if tag in self.dub2id3_dict.keys(): @@ -143,6 +146,7 @@ def write_tags(self): # media_id3.save() # except: # raise IOError('ExporterError: cannot write tags') + ''' media = id3.ID3(self.media) media.add(id3.TIT2(encoding=3, text=self.metadata['title'].decode('utf8'))) diff --git a/deefuzzer/tools/ogg.py b/deefuzzer/tools/ogg.py index bca591d..5fea49b 100644 --- a/deefuzzer/tools/ogg.py +++ b/deefuzzer/tools/ogg.py @@ -54,13 +54,14 @@ def __init__(self, media): self.options = {} self.bitrate_default = '192' self.cache_dir = os.sep + 'tmp' - self.keys2ogg = {'title': 'title', - 'artist': 'artist', - 'album': 'album', - 'date': 'date', - 'comment': 'comment', - 'genre': 'genre', - 'copyright': 'copyright', + self.keys2ogg = { + 'title': 'title', + 'artist': 'artist', + 'album': 'album', + 'date': 'date', + 'comment': 'comment', + 'genre': 'genre', + 'copyright': 'copyright' } self.info = self.ogg.info self.bitrate = int(str(self.info.bitrate)[:-3]) diff --git a/deefuzzer/tools/xmltodict.py b/deefuzzer/tools/xmltodict.py index 7d5e070..d1853be 100644 --- a/deefuzzer/tools/xmltodict.py +++ b/deefuzzer/tools/xmltodict.py @@ -7,8 +7,7 @@ def haschilds(dom): # Checks whether an element has any childs # containing real tags opposed to just text. for childnode in dom.childNodes: - if childnode.nodeName != "#text" and \ - childnode.nodeName != "#cdata-section": + if childnode.nodeName != "#text" and childnode.nodeName != "#cdata-section": return True return False diff --git a/deefuzzer/tools/xmltodict2.py b/deefuzzer/tools/xmltodict2.py index 1724ec3..ad44f03 100644 --- a/deefuzzer/tools/xmltodict2.py +++ b/deefuzzer/tools/xmltodict2.py @@ -9,6 +9,9 @@ import locale from xml.parsers import expat +""" """ + +''' # If we're in Dabo, get the default encoding. # import dabo # import dabo.lib.DesignerUtils as desUtil @@ -22,6 +25,7 @@ # if enc is None: # enc = dabo.defaultEncoding # default_encoding = enc +''' # Python seems to need to compile code with \n linesep: code_linesep = "\n" @@ -123,7 +127,6 @@ def EndElement(self, name): else: self.nodeStack = self.nodeStack[:-1] - def CharacterData(self, data): """SAX character data event handler""" if self._inCode or data.strip(): @@ -142,7 +145,6 @@ def CharacterData(self, data): element["cdata"] = "" element["cdata"] += data - def Parse(self, xml): # Create a SAX parser Parser = expat.ParserCreate() @@ -155,15 +157,18 @@ def Parse(self, xml): return self.root def ParseFromFile(self, filename): - return self.Parse(open(filename,"r").read()) + return self.Parse(open(filename, "r").read()) -def xmltodict(xml, attsToSkip=[], addCodeFile=False): +def xmltodict(xml, attsToSkip=None, addCodeFile=False): """Given an xml string or file, return a Python dictionary.""" + if not attsToSkip: + attsToSkip = [] parser = Xml2Obj() parser.attsToSkip = attsToSkip isPath = os.path.exists(xml) errmsg = "" + ret = None if eol not in xml and isPath: # argument was a file try: @@ -206,8 +211,8 @@ def escQuote(val, noEscape=False, noQuote=False): qt = '' else: qt = '"' - slsh = "\\" -# val = val.replace(slsh, slsh+slsh) + # slsh = "\\" + # val = val.replace(slsh, slsh+slsh) if not noEscape: # First escape internal ampersands. We need to double them up due to a # quirk in wxPython and the way it displays this character. @@ -258,8 +263,8 @@ def dicttoxml(dct, level=0, header=None, linesep=None): if dct.has_key("code"): if len(dct["code"].keys()): - ret += "%s%s%s" % (eol, "\t" * (level+1), eol) - methodTab = "\t" * (level+2) + ret += "%s%s%s" % (eol, "\t" * (level + 1), eol) + methodTab = "\t" * (level + 2) for mthd, cd in dct["code"].items(): # Convert \n's in the code to eol: cd = eol.join(cd.splitlines()) @@ -268,28 +273,29 @@ def dicttoxml(dct, level=0, header=None, linesep=None): if not cd.endswith(eol): cd += eol - ret += "%s<%s>%s%s%s" % (methodTab, - mthd, eol, cd, eol, - methodTab, mthd, eol) - ret += "%s%s" % ("\t" * (level+1), eol) + ret += "%s<%s>%s%s%s" % ( + methodTab, mthd, eol, + cd, eol, + methodTab, mthd, eol + ) + ret += "%s%s" % ("\t" * (level + 1), eol) if dct.has_key("properties"): if len(dct["properties"].keys()): - ret += "%s%s%s" % (eol, "\t" * (level+1), eol) - currTab = "\t" * (level+2) + ret += "%s%s%s" % (eol, "\t" * (level + 1), eol) + currTab = "\t" * (level + 2) for prop, val in dct["properties"].items(): ret += "%s<%s>%s" % (currTab, prop, eol) for propItm, itmVal in val.items(): - itmTab = "\t" * (level+3) - ret += "%s<%s>%s%s" % (itmTab, propItm, itmVal, - propItm, eol) + itmTab = "\t" * (level + 3) + ret += "%s<%s>%s%s" % (itmTab, propItm, itmVal, propItm, eol) ret += "%s%s" % (currTab, prop, eol) - ret += "%s%s" % ("\t" * (level+1), eol) + ret += "%s%s" % ("\t" * (level + 1), eol) if dct.has_key("children") and len(dct["children"]) > 0: ret += eol for child in dct["children"]: - ret += dicttoxml(child, level+1, linesep=linesep) + ret += dicttoxml(child, level + 1, linesep=linesep) indnt = "" if ret.endswith(eol): # Indent the closing tag @@ -301,8 +307,7 @@ def dicttoxml(dct, level=0, header=None, linesep=None): if level == 0: if header is None: - header = '%s' \ - % (default_encoding, eol) + header = '%s' % (default_encoding, eol) ret = header + ret return ret @@ -339,8 +344,11 @@ def flattenClassDict(cd, retDict=None): classCode = classCD.get("code", {}) classKids = classCD.get("children", []) currDict = retDict.get(classID, {}) - retDict[classID] = {"attributes": classAtts, "code": classCode, - "properties": classProps} + retDict[classID] = { + "attributes": classAtts, + "code": classCode, + "properties": classProps + } retDict[classID].update(currDict) # Now update the child objects in the dict for kid in classKids: @@ -348,8 +356,11 @@ def flattenClassDict(cd, retDict=None): else: # Not a file; most likely just a component in another class currDict = retDict.get(classID, {}) - retDict[classID] = {"attributes": atts, "code": code, - "properties": props} + retDict[classID] = { + "attributes": atts, + "code": code, + "properties": props + } retDict[classID].update(currDict) if kids: for kid in kids: @@ -381,8 +392,7 @@ def addInheritedInfo(src, super, updateCode=False): for kid in kids: addInheritedInfo(kid, super, updateCode) - - +''' # if __name__ == "__main__": # test_dict = {"name": "test", "attributes":{"path": "c:\\temp\\name", # "problemChars": "Welcome to \xc2\xae".decode("latin-1")}} @@ -392,3 +402,4 @@ def addInheritedInfo(src, super, updateCode=False): # test_dict2 = xmltodict(xml) # print "test_dict2:", test_dict2 # print "same?:", test_dict == test_dict2 +''' diff --git a/scripts/osc/osc_jingles_start.py b/scripts/osc/osc_jingles_start.py index 3f2ebf0..6e5177c 100644 --- a/scripts/osc/osc_jingles_start.py +++ b/scripts/osc/osc_jingles_start.py @@ -1,7 +1,8 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -import liblo, sys +import liblo +import sys # send all messages to port 1234 on the local machine try: diff --git a/scripts/osc/osc_jingles_stop.py b/scripts/osc/osc_jingles_stop.py index d29f721..84e3044 100644 --- a/scripts/osc/osc_jingles_stop.py +++ b/scripts/osc/osc_jingles_stop.py @@ -1,7 +1,8 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -import liblo, sys +import liblo +import sys # send all messages to port 1234 on the local machine try: diff --git a/scripts/osc/osc_player_fast.py b/scripts/osc/osc_player_fast.py index 92a60fe..fedf9db 100644 --- a/scripts/osc/osc_player_fast.py +++ b/scripts/osc/osc_player_fast.py @@ -1,7 +1,8 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -import liblo, sys +import liblo +import sys # send all messages to port 1234 on the local machine try: diff --git a/scripts/osc/osc_player_next.py b/scripts/osc/osc_player_next.py index 21a91ee..ef0e30f 100644 --- a/scripts/osc/osc_player_next.py +++ b/scripts/osc/osc_player_next.py @@ -1,7 +1,8 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -import liblo, sys +import liblo +import sys # send all messages to port 1234 on the local machine try: diff --git a/scripts/osc/osc_player_slow.py b/scripts/osc/osc_player_slow.py index 02948e0..a5761bd 100644 --- a/scripts/osc/osc_player_slow.py +++ b/scripts/osc/osc_player_slow.py @@ -1,7 +1,8 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -import liblo, sys +import liblo +import sys # send all messages to port 1234 on the local machine try: diff --git a/scripts/osc/osc_record_start.py b/scripts/osc/osc_record_start.py index 779e90b..d0b5fe5 100644 --- a/scripts/osc/osc_record_start.py +++ b/scripts/osc/osc_record_start.py @@ -1,7 +1,8 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -import liblo, sys +import liblo +import sys # send all messages to port 1234 on the local machine try: diff --git a/scripts/osc/osc_record_stop.py b/scripts/osc/osc_record_stop.py index 8056c69..0bfa651 100644 --- a/scripts/osc/osc_record_stop.py +++ b/scripts/osc/osc_record_stop.py @@ -1,7 +1,8 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -import liblo, sys +import liblo +import sys # send all messages to port 1234 on the local machine try: diff --git a/scripts/osc/osc_relay_start.py b/scripts/osc/osc_relay_start.py index 14bcb69..19c06a1 100644 --- a/scripts/osc/osc_relay_start.py +++ b/scripts/osc/osc_relay_start.py @@ -1,7 +1,8 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -import liblo, sys +import liblo +import sys # send all messages to port 1234 on the local machine try: diff --git a/scripts/osc/osc_relay_stop.py b/scripts/osc/osc_relay_stop.py index eaefe1a..d54af34 100644 --- a/scripts/osc/osc_relay_stop.py +++ b/scripts/osc/osc_relay_stop.py @@ -1,7 +1,8 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -import liblo, sys +import liblo +import sys # send all messages to port 1234 on the local machine try: diff --git a/scripts/osc/osc_twitter_start.py b/scripts/osc/osc_twitter_start.py index c298be6..b10d0cc 100644 --- a/scripts/osc/osc_twitter_start.py +++ b/scripts/osc/osc_twitter_start.py @@ -1,7 +1,8 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -import liblo, sys +import liblo +import sys # send all messages to port 1234 on the local machine try: diff --git a/scripts/osc/osc_twitter_stop.py b/scripts/osc/osc_twitter_stop.py index 3470fcc..1a33947 100644 --- a/scripts/osc/osc_twitter_stop.py +++ b/scripts/osc/osc_twitter_stop.py @@ -1,7 +1,8 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -import liblo, sys +import liblo +import sys # send all messages to port 1234 on the local machine try: diff --git a/setup.py b/setup.py index 56cccad..b0c6bf9 100644 --- a/setup.py +++ b/setup.py @@ -1,37 +1,38 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -'''The setup and build script for the library.''' +"""The setup and build script for the library.""" from setuptools import setup, find_packages -CLASSIFIERS = ['Programming Language :: Python', - 'Topic :: Internet :: WWW/HTTP :: Dynamic Content', - 'Topic :: Multimedia :: Sound/Audio', - 'Topic :: Multimedia :: Sound/Audio :: Players',] +CLASSIFIERS = [ + 'Programming Language :: Python', + 'Topic :: Internet :: WWW/HTTP :: Dynamic Content', + 'Topic :: Multimedia :: Sound/Audio', + 'Topic :: Multimedia :: Sound/Audio :: Players' +] setup( - name = "DeeFuzzer", - url = "http://github.com/yomguy/DeeFuzzer", - description = "open, light and instant media streaming tool", - long_description = open('README.rst').read(), - author = "Guillaume Pellerin", - author_email = "yomguy@parisson.com", - version = '0.7', - install_requires = [ + name="DeeFuzzer", + url="http://github.com/yomguy/DeeFuzzer", + description="open, light and instant media streaming tool", + long_description=open('README.rst').read(), + author="Guillaume Pellerin", + author_email="yomguy@parisson.com", + version='0.7', + install_requires=[ 'setuptools', 'python-shout', 'python-twitter', 'mutagen', 'pyliblo', - 'pycurl', - ], - platforms=['OS Independent'], - license='CeCILL v2', - scripts=['scripts/deefuzzer'], - classifiers = CLASSIFIERS, - packages = find_packages(), - include_package_data = True, - zip_safe = False, + 'pycurl' + ], + platforms=['OS Independent'], + license='CeCILL v2', + scripts=['scripts/deefuzzer'], + classifiers=CLASSIFIERS, + packages=find_packages(), + include_package_data=True, + zip_safe=False ) - From f9a8592ff9aa7911f879be2675bfc18802f6daec Mon Sep 17 00:00:00 2001 From: achbed Date: Mon, 26 Jan 2015 23:41:59 -0600 Subject: [PATCH 64/82] Removal of .keys() lookup where not needed (for speed) Removed the remaining calls to has_key() --- NEWS.rst | 2 +- README.rst | 2 +- deefuzzer/core.py | 28 ++++++++++++++-------------- deefuzzer/station.py | 14 +++++++------- deefuzzer/tools/logger.py | 4 ++-- deefuzzer/tools/mp3.py | 2 +- deefuzzer/tools/ogg.py | 2 +- deefuzzer/tools/xmltodict2.py | 31 ++++++++++++++++--------------- 8 files changed, 43 insertions(+), 42 deletions(-) diff --git a/NEWS.rst b/NEWS.rst index 602c6d2..b3a7e4d 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -1,5 +1,5 @@ Old news -========= +======== See README.rst for last news. diff --git a/README.rst b/README.rst index ca679be..fcccbf4 100644 --- a/README.rst +++ b/README.rst @@ -20,7 +20,7 @@ live multimedia relays or personal home radios, with metadata management and coo Features -========= +======== * MP3, OGG Vorbis file and live streaming over Internet * Full metadata encapsulation and management diff --git a/deefuzzer/core.py b/deefuzzer/core.py index 954c290..ad84834 100644 --- a/deefuzzer/core.py +++ b/deefuzzer/core.py @@ -68,7 +68,7 @@ def __init__(self, conf_file): self.conf_file = conf_file self.conf = get_conf_dict(self.conf_file) - if 'deefuzzer' not in self.conf.keys(): + if 'deefuzzer' not in self.conf: return # Get the log setting first (if possible) @@ -145,12 +145,12 @@ def create_stations_fromfolder(self): """Scan a folder for subfolders containing media, and make stations from them all.""" options = self.watchfolder - if 'folder' not in options.keys(): + if 'folder' not in options: # We have no folder specified. Bail. return if self.mainLoop: - if 'livecreation' not in options.keys(): + if 'livecreation' not in options: # We have no folder specified. Bail. return @@ -166,9 +166,9 @@ def create_stations_fromfolder(self): # This makes the log file a lot more verbose. Commented out since we report on new stations anyway. # self._info('Scanning folder ' + folder + ' for stations') - if 'infos' not in options.keys(): + if 'infos' not in options: options['infos'] = {} - if 'short_name' not in options['infos'].keys(): + if 'short_name' not in options['infos']: options['infos']['short_name'] = '[name]' files = os.listdir(folder) @@ -181,9 +181,9 @@ def create_stations_fromfolder(self): def station_exists(self, name): try: for s in self.station_settings: - if 'infos' not in s.keys(): + if 'infos' not in s: continue - if 'short_name' not in s['infos'].keys(): + if 'short_name' not in s['infos']: continue if s['infos']['short_name'] == name: return True @@ -204,7 +204,7 @@ def create_station(self, folder, options): for i in options.keys(): if 'folder' not in i: s[i] = replace_all(options[i], d) - if 'media' not in s.keys(): + if 'media' not in s: s['media'] = {} s['media']['source'] = folder @@ -241,7 +241,7 @@ def load_station_config(self, file): self._info('Loading station config file ' + file) stationdef = get_conf_dict(file) if isinstance(stationdef, dict): - if 'station' in stationdef.keys(): + if 'station' in stationdef: if isinstance(stationdef['station'], dict): self.add_station(stationdef['station']) elif isinstance(stationdef['station'], list): @@ -272,14 +272,14 @@ def run(self): for i in range(0, ns_new): name = '' try: - if 'station_name' in self.station_settings[i].keys(): + if 'station_name' in self.station_settings[i]: name = self.station_settings[i]['station_name'] - if 'retries' not in self.station_settings[i].keys(): + if 'retries' not in self.station_settings[i]: self.station_settings[i]['retries'] = 0 try: - if 'station_instance' in self.station_settings[i].keys(): + if 'station_instance' in self.station_settings[i]: # Check for station running here if self.station_settings[i]['station_instance'].isAlive(): # Station exists and is alive. Don't recreate. @@ -288,7 +288,7 @@ def run(self): if self.maxretry >= 0 and self.station_settings[i]['retries'] <= self.maxretry: # Station passed the max retries count is will not be reloaded - if 'station_stop_logged' not in self.station_settings[i].keys(): + if 'station_stop_logged' not in self.station_settings[i]: self._err('Station ' + name + ' is stopped and will not be restarted.') self.station_settings[i]['station_stop_logged'] = True continue @@ -312,7 +312,7 @@ def run(self): if name == '': name = 'Station ' + str(i) - if 'info' in self.station_settings[i].keys(): + if 'info' in self.station_settings[i]: if 'short_name' in self.station_settings[i]['infos']: name = self.station_settings[i]['infos']['short_name'] y = 1 diff --git a/deefuzzer/station.py b/deefuzzer/station.py index 4e34f09..4cef3fa 100644 --- a/deefuzzer/station.py +++ b/deefuzzer/station.py @@ -109,15 +109,15 @@ def __init__(self, station, q, logqueue, m3u): self.base_directory = self.station['base_dir'].strip() # Media - if 'm3u' in self.station['media'].keys(): + if 'm3u' in self.station['media']: if not self.station['media']['m3u'].strip() == '': self.media_source = self._path_add_base(self.station['media']['m3u']) - if 'dir' in self.station['media'].keys(): + if 'dir' in self.station['media']: if not self.station['media']['dir'].strip() == '': self.media_source = self._path_add_base(self.station['media']['dir']) - if 'source' in self.station['media'].keys(): + if 'source' in self.station['media']: if not self.station['media']['source'].strip() == '': self.media_source = self._path_add_base(self.station['media']['source']) @@ -129,16 +129,16 @@ def __init__(self, station, q, logqueue, m3u): self.voices = int(self.station['media']['voices']) # Server - if 'mountpoint' in self.station['server'].keys(): + if 'mountpoint' in self.station['server']: self.mountpoint = self.station['server']['mountpoint'] - elif 'short_name' in self.station['infos'].keys(): + elif 'short_name' in self.station['infos']: self.mountpoint = self.station['infos']['short_name'] else: self.mountpoint = 'default' self.short_name = self.mountpoint - if 'appendtype' in self.station['server'].keys(): + if 'appendtype' in self.station['server']: self.appendtype = int(self.station['server']['appendtype']) if 'type' in self.station['server']: @@ -730,7 +730,7 @@ def set_webm_read_mode(self): def update_twitter_current(self): if not self.__twitter_should_update(): return - artist_names = self.artist.split(' ') + # artist_names = self.artist.split(' ') # artist_tags = ' #'.join(list(set(artist_names) - {'&', '-'})) message = '%s %s on #%s' % (self.prefix, self.song, self.short_name) tags = '#' + ' #'.join(self.twitter_tags) diff --git a/deefuzzer/tools/logger.py b/deefuzzer/tools/logger.py index ae46e2b..e6b7107 100644 --- a/deefuzzer/tools/logger.py +++ b/deefuzzer/tools/logger.py @@ -38,10 +38,10 @@ def run(self): if not isinstance(msg, dict): self.logger.write_error(str(msg)) else: - if 'msg' not in msg.keys(): + if 'msg' not in msg: continue - if 'level' in msg.keys(): + if 'level' in msg: if msg['level'] == 'info': self.logger.write_info(msg['msg']) else: diff --git a/deefuzzer/tools/mp3.py b/deefuzzer/tools/mp3.py index eb3f61e..f0e52c1 100644 --- a/deefuzzer/tools/mp3.py +++ b/deefuzzer/tools/mp3.py @@ -133,7 +133,7 @@ def write_tags(self): ''' # media_id3 = id3.ID3(self.media) # for tag in self.metadata.keys(): - # if tag in self.dub2id3_dict.keys(): + # if tag in self.dub2id3_dict: # frame_text = self.dub2id3_dict[tag] # value = self.metadata[tag] # frame = mutagen.id3.Frames[frame_text](3,value) diff --git a/deefuzzer/tools/ogg.py b/deefuzzer/tools/ogg.py index 5fea49b..a970f8a 100644 --- a/deefuzzer/tools/ogg.py +++ b/deefuzzer/tools/ogg.py @@ -145,7 +145,7 @@ def get_args(self, options=None): for tag in self.metadata.keys(): value = clean_word(self.metadata[tag]) args.append('-c %s="%s"' % (tag, value)) - if tag in self.dub2args_dict.keys(): + if tag in self.dub2args_dict: arg = self.dub2args_dict[tag] args.append('-c %s="%s"' % (arg, value)) diff --git a/deefuzzer/tools/xmltodict2.py b/deefuzzer/tools/xmltodict2.py index ad44f03..bf0ea46 100644 --- a/deefuzzer/tools/xmltodict2.py +++ b/deefuzzer/tools/xmltodict2.py @@ -55,7 +55,7 @@ def StartElement(self, name, attributes): # This is code for the parent element self._inCode = True parent = self.nodeStack[-1] - if not parent.has_key("code"): + if "code" not in parent: parent["code"] = {} self._codeDict = parent["code"] @@ -65,7 +65,7 @@ def StartElement(self, name, attributes): self._propName = "" self._propData = "" parent = self.nodeStack[-1] - if not parent.has_key("properties"): + if "properties" not in parent: parent["properties"] = {} self._propDict = parent["properties"] @@ -84,14 +84,14 @@ def StartElement(self, name, attributes): element = {"name": name.encode()} if len(attributes) > 0: for att in self.attsToSkip: - if attributes.has_key(att): + if att in attributes: del attributes[att] element["attributes"] = attributes # Push element onto the stack and make it a child of parent if len(self.nodeStack) > 0: parent = self.nodeStack[-1] - if not parent.has_key("children"): + if "children" not in parent: parent["children"] = [] parent["children"].append(element) else: @@ -141,7 +141,7 @@ def CharacterData(self, data): self._propData += data else: element = self.nodeStack[-1] - if not element.has_key("cdata"): + if "cdata" not in element: element["cdata"] = "" element["cdata"] += data @@ -245,7 +245,7 @@ def dicttoxml(dct, level=0, header=None, linesep=None): att = "" ret = "" - if dct.has_key("attributes"): + if "attributes" in dct: for key, val in dct["attributes"].items(): # Some keys are already handled. noEscape = key in ("sizerInfo",) @@ -253,15 +253,15 @@ def dicttoxml(dct, level=0, header=None, linesep=None): att += " %s=%s" % (key, val) ret += "%s<%s%s" % ("\t" * level, dct["name"], att) - if (not dct.has_key("cdata") and not dct.has_key("children") - and not dct.has_key("code") and not dct.has_key("properties")): + if ("cdata" not in dct and "children" not in dct + and "code" not in dct and "properties" not in dct): ret += " />%s" % eol else: ret += ">" - if dct.has_key("cdata"): + if "cdata" in dct: ret += "%s" % dct["cdata"].replace("<", "<") - if dct.has_key("code"): + if "code" in dct: if len(dct["code"].keys()): ret += "%s%s%s" % (eol, "\t" * (level + 1), eol) methodTab = "\t" * (level + 2) @@ -280,7 +280,7 @@ def dicttoxml(dct, level=0, header=None, linesep=None): ) ret += "%s%s" % ("\t" * (level + 1), eol) - if dct.has_key("properties"): + if "properties" in dct: if len(dct["properties"].keys()): ret += "%s%s%s" % (eol, "\t" * (level + 1), eol) currTab = "\t" * (level + 2) @@ -292,10 +292,11 @@ def dicttoxml(dct, level=0, header=None, linesep=None): ret += "%s%s" % (currTab, prop, eol) ret += "%s%s" % ("\t" * (level + 1), eol) - if dct.has_key("children") and len(dct["children"]) > 0: - ret += eol - for child in dct["children"]: - ret += dicttoxml(child, level + 1, linesep=linesep) + if "children" in dct: + if len(dct["children"]) > 0: + ret += eol + for child in dct["children"]: + ret += dicttoxml(child, level + 1, linesep=linesep) indnt = "" if ret.endswith(eol): # Indent the closing tag From 1cf88da68b68e8d06738e51f70c860ce5d355ee5 Mon Sep 17 00:00:00 2001 From: achbed Date: Tue, 27 Jan 2015 12:11:13 -0600 Subject: [PATCH 65/82] Started refactoring to remove shadowed parameter warnings Fix for bitrate calcuation (now divide by 1024 instead of chop string) - addresses #55 Signed-off-by: achbed --- deefuzzer/tools/logger.py | 8 ++++---- deefuzzer/tools/mp3.py | 8 ++++++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/deefuzzer/tools/logger.py b/deefuzzer/tools/logger.py index e6b7107..a07e302 100644 --- a/deefuzzer/tools/logger.py +++ b/deefuzzer/tools/logger.py @@ -8,9 +8,9 @@ class Logger: """A logging object""" - def __init__(self, file): + def __init__(self, filepath): self.logger = logging.getLogger('myapp') - self.hdlr = logging.FileHandler(file) + self.hdlr = logging.FileHandler(filepath) self.formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s') self.hdlr.setFormatter(self.formatter) self.logger.addHandler(self.hdlr) @@ -26,9 +26,9 @@ def write_error(self, message): class QueueLogger(Thread): """A queue-based logging object""" - def __init__(self, file, q): + def __init__(self, filepath, q): Thread.__init__(self) - self.logger = Logger(file) + self.logger = Logger(filepath) self.q = q def run(self): diff --git a/deefuzzer/tools/mp3.py b/deefuzzer/tools/mp3.py index f0e52c1..4581300 100644 --- a/deefuzzer/tools/mp3.py +++ b/deefuzzer/tools/mp3.py @@ -58,7 +58,7 @@ def __init__(self, media): self.item_id = '' self.source = self.media self.options = {} - self.bitrate_default = '192' + self.bitrate_default = 192 self.cache_dir = os.sep + 'tmp' self.keys2id3 = { 'title': 'TIT2', @@ -72,7 +72,11 @@ def __init__(self, media): } self.mp3 = MP3(self.media, ID3=EasyID3) self.info = self.mp3.info - self.bitrate = int(str(self.info.bitrate)[:-3]) + self.bitrate = self.bitrate_default + try: + self.bitrate = int(self.info.bitrate / 1024) + except: + pass self.length = datetime.timedelta(0, self.info.length) try: self.metadata = self.get_file_metadata() From c68f4a42b4e73e2847a9fab288a74091587fe16f Mon Sep 17 00:00:00 2001 From: achbed Date: Tue, 27 Jan 2015 17:13:47 -0600 Subject: [PATCH 66/82] New MediaBase class for common media object functionality Refactored MP3, OGG, WebM classes to use MediaBase Station now uses common MediaBase functions to get artist, title, and song info --- deefuzzer/station.py | 106 +++++++++++++---------------- deefuzzer/tools/__init__.py | 1 + deefuzzer/tools/mediabase.py | 127 +++++++++++++++++++++++++++++++++++ deefuzzer/tools/mp3.py | 58 +++++----------- deefuzzer/tools/ogg.py | 83 +++++++++-------------- deefuzzer/tools/webm.py | 13 ++-- 6 files changed, 230 insertions(+), 158 deletions(-) create mode 100644 deefuzzer/tools/mediabase.py diff --git a/deefuzzer/station.py b/deefuzzer/station.py index 4cef3fa..1e2ae72 100644 --- a/deefuzzer/station.py +++ b/deefuzzer/station.py @@ -95,6 +95,8 @@ def __init__(self, station, q, logqueue, m3u): self.logqueue = logqueue self.m3u = m3u + self.current_media_obj = MediaBase() + if 'station_statusfile' in self.station: self.statusfile = station['station_statusfile'] try: @@ -452,24 +454,12 @@ def tweet(self): if len(self.new_tracks): new_tracks_objs = self.media_to_objs(self.new_tracks) for media_obj in new_tracks_objs: - title = '' - artist = '' - if 'title' in media_obj.metadata: - title = media_obj.metadata['title'] - if 'artist' in media_obj.metadata: - artist = media_obj.metadata['artist'] - if not (title or artist): - song = str(media_obj.file_name) - else: - song = artist + ' - ' + title - - song = song.encode('utf-8') - artist = artist.encode('utf-8') + title, artist, song = self.get_songmeta(media_obj) artist_names = artist.split(' ') artist_tags = ' #'.join(list(set(artist_names) - {'&', '-'})) - message = '#NEWTRACK ! %s #%s on #%s RSS: ' % \ - (song.replace('_', ' '), artist_tags, self.short_name) + message = '#NEWTRACK ! %s %s on #%s' % \ + (song, artist_tags.strip(), self.short_name) message = message[:113] + self.feeds_url self.update_twitter(message) @@ -547,30 +537,20 @@ def get_next_media(self): def media_to_objs(self, media_list): media_objs = [] for media in media_list: + file_meta = MediaBase() file_name, file_title, file_ext = get_file_info(media) self.q.get(1) try: if file_ext.lower() == 'mp3' or mimetypes.guess_type(media)[0] == 'audio/mpeg': - try: - file_meta = Mp3(media) - except: - continue + file_meta = Mp3(media) elif file_ext.lower() == 'ogg' or mimetypes.guess_type(media)[0] == 'audio/ogg': - try: - file_meta = Ogg(media) - except: - continue + file_meta = Ogg(media) elif file_ext.lower() == 'webm' or mimetypes.guess_type(media)[0] == 'video/webm': - try: - file_meta = WebM(media) - except: - continue - if self.feeds_showfilename: - file_meta.metadata['filename'] = file_name.decode("utf-8") # decode needed for some weird filenames - if self.feeds_showfilepath: - file_meta.metadata['filepath'] = media.decode("utf-8") # decode needed for some weird filenames + file_meta = WebM(media) except: pass + file_meta.metadata['filename'] = file_name.decode("utf-8") # decode needed for some weird filenames + file_meta.metadata['filepath'] = media.decode("utf-8") # decode needed for some weird filenames self.q.task_done() media_objs.append(file_meta) return media_objs @@ -602,18 +582,16 @@ def update_feeds(self, media_list, rss_file, sub_title): for key in media.metadata.keys(): if media.metadata[key] != '': + if key == 'filepath' and not self.feeds_showfilepath: + continue + if key == 'filename' and not self.feeds_showfilename: + continue media_description += media_description_item % (key.capitalize(), media.metadata[key]) json_item[key] = media.metadata[key] media_description += '
%s: %s
' - title = media.metadata['title'] - artist = media.metadata['artist'] - if not (title or artist): - song = str(media.file_title) - else: - song = artist + ' - ' + title - + title, artist, song = self.get_songmeta(media) media_absolute_playtime += media.length if self.feeds_enclosure == '1': @@ -674,11 +652,7 @@ def update_twitter(self, message): def set_relay_mode(self): self.prefix = '#nowplaying #LIVE' - self.title = self.channel.description.encode('utf-8') - self.artist = self.relay_author.encode('utf-8') - self.title = self.title.replace('_', ' ') - self.artist = self.artist.replace('_', ' ') - self.song = self.artist + ' - ' + self.title + self.get_currentsongmeta() if self.type == 'stream-m': relay = URLReader(self.relay_url) @@ -688,28 +662,40 @@ def set_relay_mode(self): else: self.stream = self.player.relay_read() - def set_read_mode(self): - self.prefix = '#nowplaying' - self.current_media_obj = self.media_to_objs([self.media]) + def get_songmeta(self, mediaobj): + title = "" + artist = "" + song = "" + try: - self.title = self.current_media_obj[0].metadata['title'] - self.artist = self.current_media_obj[0].metadata['artist'] + title = mediaobj.get_title() + artist = mediaobj.get_artist() + song = mediaobj.get_song(True) except: - self.title = 'title' - self.artist = 'artist' + pass - self.title = self.title.replace('_', ' ') - self.artist = self.artist.replace('_', ' ') + return title, artist, song - if not (self.title or self.artist): - song = str(self.current_media_obj[0].file_name) - else: - song = self.artist + ' - ' + self.title + def get_currentsongmeta(self): + self.title = "" + self.artist = "" + self.song = "" + self.current_media_obj = MediaBase() + + try: + m = self.media_to_objs([self.media]) + self.current_media_obj = m[0] + except: + pass + + self.title, self.artist, self.song = self.get_songmeta(self.current_media_obj) + + def set_read_mode(self): + self.prefix = '#nowplaying' + self.get_currentsongmeta() - self.song = song.encode('utf-8') - self.artist = self.artist.encode('utf-8') self.metadata_file = self.metadata_dir + os.sep + self.current_media_obj[0].file_name + '.xml' - self.update_feeds(self.current_media_obj, self.feeds_current_file, '(currently playing)') + self.update_feeds([self.current_media_obj], self.feeds_current_file, '(currently playing)') self._info('DeeFuzzing: id = %s, name = %s' % (self.id, self.current_media_obj[0].file_name)) self.player.set_media(self.media) @@ -730,8 +716,6 @@ def set_webm_read_mode(self): def update_twitter_current(self): if not self.__twitter_should_update(): return - # artist_names = self.artist.split(' ') - # artist_tags = ' #'.join(list(set(artist_names) - {'&', '-'})) message = '%s %s on #%s' % (self.prefix, self.song, self.short_name) tags = '#' + ' #'.join(self.twitter_tags) message = message + ' ' + tags diff --git a/deefuzzer/tools/__init__.py b/deefuzzer/tools/__init__.py index 6dc964d..276c198 100644 --- a/deefuzzer/tools/__init__.py +++ b/deefuzzer/tools/__init__.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- from xmltodict import * from PyRSS2Gen import * +from mediabase import * from mp3 import * from ogg import * from webm import * diff --git a/deefuzzer/tools/mediabase.py b/deefuzzer/tools/mediabase.py new file mode 100644 index 0000000..4474e63 --- /dev/null +++ b/deefuzzer/tools/mediabase.py @@ -0,0 +1,127 @@ +__author__ = 'Dennis Wallace' + +import tempfile + +class MediaBase(object): + """Base Media class. All media objects should inherit from this class + to allow common functions to be used in core code. See MP3 and OGG classes + for examples on how to configure a subclass.""" + + def __init__(self): + object.__init__(self) + + # Set the following five values in an inherited subclass. + + # A text string describing this media type + self.description = '' + + # A text string declaring the MIME Type for this media type + self.mime_type = '' + + # A text string declaring the common file extension for this media type + self.extension = '' + + # A text string declaring the media format. The self.format property + # should be unique across all subclasses inherited from MediaBase. + self.format = '' + + # tagdata contains a dictionary of tags to use to gather metadata from the sourceobj + self.tagdata = {} + + self.media = None + self.item_id = '' + self.source = None + self.options = {} + self.bitrate_default = 0 + self.info = None + self.bitrate = 0 + self.length = 0 + self.metadata = { + 'title': '', + 'artist': '', + 'album': '', + 'date': '', + 'comment': '', + 'country': '', + 'genre': '', + 'copyright': '' + } + + # sourceobj contains the metadata information for the referenced object + self.sourceobj = {} + + self.media_info = [] + self.file_name = '' + self.file_title = '' + self.file_ext = '' + self.size = 0 + + # A more cross-platform way to do this + self.cache_dir = tempfile.gettempdir() + + def get_format(self): + """Gets the format string of the media type""" + return self.format + + def get_file_extension(self): + """Gets the actual file extension string of the media""" + return self.file_ext + + def get_mime_type(self): + """Gets the MIME Type string for this media type""" + return self.mime_type + + def get_description(self): + """Gets the description string for this media type""" + return self.description + + def set_cache_dir(self, path): + """Sets an alternate location for temporary cache files used in this media object""" + self.cache_dir = path + + def get_file_metadata(self): + """Returns the metadata for the media, filtered by the tagdata dictionary for this media type""" + metadata = {} + for key in self.tagdata.keys(): + metadata[key] = '' + try: + metadata[key] = self.sourceobj[key][0] + except: + try: + if self.tagdata[key] != '': + metadata[key] = self.sourceobj[self.tagdata[key]][0] + except: + pass + return metadata + + def __get_metadata_value(self, key, clean=False): + """Returns a metadata value for a give key. If clean is True, then the resulting string will + be cleaned before it is returned. If the key does not exist, an empty string is returned.""" + if key not in self.metadata: + return '' + r = self.metadata[key] + if clean: + r = r.replace('_',' ').strip() + return r.encode('utf-8') + + def get_title(self): + """Returns the cleaned title for this media""" + return self.__get_metadata_value('title', True) + + def get_artist(self): + """Returns the cleaned artist for this media""" + return self.__get_metadata_value('artist', True) + + def get_song(self, usefn=True): + """Returns a string in the form "artist - title" for this media. If either artist or title are blank, + only the non-blank field is returned. If both fields are blank, and the usefn parameter is True, then + the filename is returned instead. Otherwise, an empty string is returned.""" + a = self.__get_metadata_value('artist', True) + t = self.__get_metadata_value('title', True) + if len(a) == 0 and len(t) == 0 and usefn: + a = self.file_name.encode('utf-8') + r = a + if len(a) > 0 and len(t) > 0: + r += ' - ' + r += t + return r diff --git a/deefuzzer/tools/mp3.py b/deefuzzer/tools/mp3.py index 4581300..fbb1020 100644 --- a/deefuzzer/tools/mp3.py +++ b/deefuzzer/tools/mp3.py @@ -50,17 +50,21 @@ EasyID3.RegisterTXXXKey("country", "COUNTRY") -class Mp3: +class Mp3(MediaBase): """A MP3 file object""" - def __init__(self, media): - self.media = media - self.item_id = '' + def __init__(self, newmedia): + MediaBase.__init__(self) + + self.description = "MPEG audio Layer III" + self.mime_type = 'audio/mpeg' + self.extension = 'mp3' + self.format = 'MP3' + + self.media = newmedia self.source = self.media - self.options = {} self.bitrate_default = 192 - self.cache_dir = os.sep + 'tmp' - self.keys2id3 = { + self.tagdata = { 'title': 'TIT2', 'artist': 'TPE1', 'album': 'TALB', @@ -70,8 +74,8 @@ def __init__(self, media): 'genre': 'TCON', 'copyright': 'TCOP' } - self.mp3 = MP3(self.media, ID3=EasyID3) - self.info = self.mp3.info + self.sourceobj = MP3(self.media, ID3=EasyID3) + self.info = self.sourceobj.info self.bitrate = self.bitrate_default try: self.bitrate = int(self.info.bitrate / 1024) @@ -92,47 +96,19 @@ def __init__(self, media): 'copyright': '' } - self.description = self.get_description() - self.mime_type = self.get_mime_type() self.media_info = get_file_info(self.media) self.file_name = self.media_info[0] self.file_title = self.media_info[1] self.file_ext = self.media_info[2] - self.extension = self.get_file_extension() - self.size = os.path.getsize(media) - # self.args = self.get_args() - - def get_format(self): - return 'MP3' - - def get_file_extension(self): - return 'mp3' - - def get_mime_type(self): - return 'audio/mpeg' - - def get_description(self): - return "MPEG audio Layer III" - - def get_file_metadata(self): - metadata = {} - for key in self.keys2id3.keys(): - try: - metadata[key] = self.mp3[key][0] - except: - try: - metadata[key] = self.mp3[self.keys2id3[key]][0] - except: - metadata[key] = '' - return metadata + self.size = os.path.getsize(mediabase) def write_tags(self): """Write all ID3v2.4 tags by mapping dub2id3_dict dictionnary with the respect of mutagen classes and methods""" - self.mp3.add_tags() - self.mp3.tags['TIT2'] = id3.TIT2(encoding=2, text=u'text') - self.mp3.save() + self.sourceobj.add_tags() + self.sourceobj.tags['TIT2'] = id3.TIT2(encoding=2, text=u'text') + self.sourceobj.save() ''' # media_id3 = id3.ID3(self.media) diff --git a/deefuzzer/tools/ogg.py b/deefuzzer/tools/ogg.py index a970f8a..c9834c8 100644 --- a/deefuzzer/tools/ogg.py +++ b/deefuzzer/tools/ogg.py @@ -43,55 +43,45 @@ from utils import * -class Ogg: +class Ogg(MediaBase): """An OGG file object""" def __init__(self, media): + MediaBase.__init__(self) + + self.description = "OGG Vorbis" + self.mime_type = 'audio/ogg' + self.extension = 'ogg' + self.format = 'OGG' + self.media = media - self.ogg = OggVorbis(self.media) - self.item_id = '' + self.sourceobj = OggVorbis(self.media) self.source = self.media - self.options = {} self.bitrate_default = '192' - self.cache_dir = os.sep + 'tmp' - self.keys2ogg = { - 'title': 'title', - 'artist': 'artist', - 'album': 'album', - 'date': 'date', - 'comment': 'comment', - 'genre': 'genre', - 'copyright': 'copyright' + + self.tagdata = { + 'title': '', + 'artist': '', + 'album': '', + 'date': '', + 'comment': '', + 'genre': '', + 'copyright': '' } - self.info = self.ogg.info + + self.info = self.sourceobj.info self.bitrate = int(str(self.info.bitrate)[:-3]) self.length = datetime.timedelta(0, self.info.length) self.metadata = self.get_file_metadata() - self.description = self.get_description() - self.mime_type = self.get_mime_type() self.media_info = get_file_info(self.media) self.file_name = self.media_info[0] self.file_title = self.media_info[1] self.file_ext = self.media_info[2] - self.extension = self.get_file_extension() self.size = os.path.getsize(media) - # self.args = self.get_args() - - def get_format(self): - return 'OGG' - - def get_file_extension(self): - return 'ogg' - - def get_mime_type(self): - return 'audio/ogg' - - def get_description(self): - return 'OGG Vorbis' def get_file_info(self): try: - file_out1, file_out2 = os.popen4('ogginfo "' + self.dest + '"') + file_out1, file_out2 = os.popen4('ogginfo "' + self.source + '"') info = [] for line in file_out2.readlines(): info.append(clean_word(line[:-1])) @@ -100,31 +90,21 @@ def get_file_info(self): except: raise IOError('ExporterError: file does not exist.') - def set_cache_dir(self, path): - self.cache_dir = path - - def get_file_metadata(self): - metadata = {} - for key in self.keys2ogg.keys(): - try: - metadata[key] = self.ogg[key][0] - except: - metadata[key] = '' - return metadata - def decode(self): + if not self.item_id: + raise IOError('ExporterError: Required item_id parameter not set.') try: - os.system('oggdec -o "' + self.cache_dir + os.sep + self.item_id + - '.wav" "' + self.source + '"') - return self.cache_dir + os.sep + self.item_id + '.wav' + p = os.path.join(self.cache_dir, (self.item_id + '.wav')) + os.system('oggdec -o "' + p + '" "' + self.source + '"') + return p except: raise IOError('ExporterError: decoder is not compatible.') def write_tags(self): # self.ogg.add_tags() for tag in self.metadata.keys(): - self.ogg[tag] = str(self.metadata[tag]) - self.ogg.save() + self.sourceobj[tag] = str(self.metadata[tag]) + self.sourceobj.save() def get_args(self, options=None): """Get process options and return arguments for the encoder""" @@ -145,8 +125,9 @@ def get_args(self, options=None): for tag in self.metadata.keys(): value = clean_word(self.metadata[tag]) args.append('-c %s="%s"' % (tag, value)) - if tag in self.dub2args_dict: - arg = self.dub2args_dict[tag] - args.append('-c %s="%s"' % (arg, value)) + if tag in self.tagdata: + arg = self.tagdata[tag] + if arg: + args.append('-c %s="%s"' % (arg, value)) return args diff --git a/deefuzzer/tools/webm.py b/deefuzzer/tools/webm.py index 389149d..fc7d25e 100644 --- a/deefuzzer/tools/webm.py +++ b/deefuzzer/tools/webm.py @@ -42,15 +42,18 @@ from utils import * -class WebM(object): +class WebM(MediaBase): """An WebM file object""" def __init__(self, media): - self.media = media - self.item_id = '' - self.source = self.media + MediaBase.__init__(self) + + self.description = "WebM" self.mime_type = 'video/webm' self.extension = 'webm' - self.description = 'WebM' + self.format = 'WebM' + + self.media = media + self.source = self.media self.metadata = {} self.file_name, self.file_title, self.file_ext = get_file_info(media) From 4bb4f649cedb320df70dd05e98e21c863773b290 Mon Sep 17 00:00:00 2001 From: achbed Date: Tue, 27 Jan 2015 23:34:37 -0600 Subject: [PATCH 67/82] Additional cleanup of MediaBase.metadata instances, calls, and methods Fixed exception failure in URLReader class Fixed two outstanding MediaBase reference issues in station.py Signed-off-by: achbed --- deefuzzer/player.py | 6 ++++-- deefuzzer/station.py | 4 ++-- deefuzzer/tools/mediabase.py | 41 +++++++++++++++++------------------- deefuzzer/tools/mp3.py | 16 ++------------ deefuzzer/tools/ogg.py | 4 ++-- deefuzzer/tools/webm.py | 1 - 6 files changed, 29 insertions(+), 43 deletions(-) diff --git a/deefuzzer/player.py b/deefuzzer/player.py index 27bab04..ac7fe06 100644 --- a/deefuzzer/player.py +++ b/deefuzzer/player.py @@ -37,6 +37,7 @@ # Author: Guillaume Pellerin from relay import * +import time class Player: @@ -120,7 +121,8 @@ def read_callback(self, size): class URLReader: def __init__(self, relay): - self.relay = urllib.urlopen(relay) + self.__relayparam = relay + self.relay = urllib.urlopen(self.__relayparam) self.rec_mode = 0 def set_recorder(self, recorder, mode=1): @@ -135,7 +137,7 @@ def read_callback(self, size): except: while True: try: - self.relay = urllib.urlopen(relay) + self.relay = urllib.urlopen(self.__relayparam) chunk = self.relay.read(size) break except: diff --git a/deefuzzer/station.py b/deefuzzer/station.py index 1e2ae72..6272434 100644 --- a/deefuzzer/station.py +++ b/deefuzzer/station.py @@ -694,10 +694,10 @@ def set_read_mode(self): self.prefix = '#nowplaying' self.get_currentsongmeta() - self.metadata_file = self.metadata_dir + os.sep + self.current_media_obj[0].file_name + '.xml' + self.metadata_file = self.metadata_dir + os.sep + self.current_media_obj.file_name + '.xml' self.update_feeds([self.current_media_obj], self.feeds_current_file, '(currently playing)') self._info('DeeFuzzing: id = %s, name = %s' - % (self.id, self.current_media_obj[0].file_name)) + % (self.id, self.current_media_obj.file_name)) self.player.set_media(self.media) self.q.get(1) diff --git a/deefuzzer/tools/mediabase.py b/deefuzzer/tools/mediabase.py index 4474e63..ef71132 100644 --- a/deefuzzer/tools/mediabase.py +++ b/deefuzzer/tools/mediabase.py @@ -36,16 +36,6 @@ def __init__(self): self.info = None self.bitrate = 0 self.length = 0 - self.metadata = { - 'title': '', - 'artist': '', - 'album': '', - 'date': '', - 'comment': '', - 'country': '', - 'genre': '', - 'copyright': '' - } # sourceobj contains the metadata information for the referenced object self.sourceobj = {} @@ -55,6 +45,7 @@ def __init__(self): self.file_title = '' self.file_ext = '' self.size = 0 + self.metadata = {} # A more cross-platform way to do this self.cache_dir = tempfile.gettempdir() @@ -79,22 +70,28 @@ def set_cache_dir(self, path): """Sets an alternate location for temporary cache files used in this media object""" self.cache_dir = path - def get_file_metadata(self): - """Returns the metadata for the media, filtered by the tagdata dictionary for this media type""" - metadata = {} + def get_file_metadata(self, clear_cache=False): + """Returns the metadata for the media, filtered by the tagdata dictionary for this media type. Return value is + read from cache if possible (or unless clear_cache is set to True)""" + if not self.metadata or clear_cache: + self.__read_file_metadata() + return self.metadata + + def __read_file_metadata(self): + """Reads the metadata for the media, filtered by the tagdata dictionary for this media type""" + self.metadata = {} for key in self.tagdata.keys(): - metadata[key] = '' + self.metadata[key] = '' try: - metadata[key] = self.sourceobj[key][0] + self.metadata[key] = self.sourceobj[key][0] except: try: if self.tagdata[key] != '': - metadata[key] = self.sourceobj[self.tagdata[key]][0] + self.metadata[key] = self.sourceobj[self.tagdata[key]][0] except: pass - return metadata - def __get_metadata_value(self, key, clean=False): + def get_metadata_value(self, key, clean=False): """Returns a metadata value for a give key. If clean is True, then the resulting string will be cleaned before it is returned. If the key does not exist, an empty string is returned.""" if key not in self.metadata: @@ -106,18 +103,18 @@ def __get_metadata_value(self, key, clean=False): def get_title(self): """Returns the cleaned title for this media""" - return self.__get_metadata_value('title', True) + return self.get_metadata_value('title', True) def get_artist(self): """Returns the cleaned artist for this media""" - return self.__get_metadata_value('artist', True) + return self.get_metadata_value('artist', True) def get_song(self, usefn=True): """Returns a string in the form "artist - title" for this media. If either artist or title are blank, only the non-blank field is returned. If both fields are blank, and the usefn parameter is True, then the filename is returned instead. Otherwise, an empty string is returned.""" - a = self.__get_metadata_value('artist', True) - t = self.__get_metadata_value('title', True) + a = self.get_metadata_value('artist', True) + t = self.get_metadata_value('title', True) if len(a) == 0 and len(t) == 0 and usefn: a = self.file_name.encode('utf-8') r = a diff --git a/deefuzzer/tools/mp3.py b/deefuzzer/tools/mp3.py index fbb1020..a1cafd0 100644 --- a/deefuzzer/tools/mp3.py +++ b/deefuzzer/tools/mp3.py @@ -82,25 +82,13 @@ def __init__(self, newmedia): except: pass self.length = datetime.timedelta(0, self.info.length) - try: - self.metadata = self.get_file_metadata() - except: - self.metadata = { - 'title': '', - 'artist': '', - 'album': '', - 'date': '', - 'comment': '', - 'country': '', - 'genre': '', - 'copyright': '' - } + self.__read_file_metadata() self.media_info = get_file_info(self.media) self.file_name = self.media_info[0] self.file_title = self.media_info[1] self.file_ext = self.media_info[2] - self.size = os.path.getsize(mediabase) + self.size = os.path.getsize(self.media) def write_tags(self): """Write all ID3v2.4 tags by mapping dub2id3_dict dictionnary with the diff --git a/deefuzzer/tools/ogg.py b/deefuzzer/tools/ogg.py index c9834c8..93ec0d8 100644 --- a/deefuzzer/tools/ogg.py +++ b/deefuzzer/tools/ogg.py @@ -72,12 +72,12 @@ def __init__(self, media): self.info = self.sourceobj.info self.bitrate = int(str(self.info.bitrate)[:-3]) self.length = datetime.timedelta(0, self.info.length) - self.metadata = self.get_file_metadata() + self.__read_file_metadata() self.media_info = get_file_info(self.media) self.file_name = self.media_info[0] self.file_title = self.media_info[1] self.file_ext = self.media_info[2] - self.size = os.path.getsize(media) + self.size = os.path.getsize(self.media) def get_file_info(self): try: diff --git a/deefuzzer/tools/webm.py b/deefuzzer/tools/webm.py index fc7d25e..f39eff0 100644 --- a/deefuzzer/tools/webm.py +++ b/deefuzzer/tools/webm.py @@ -55,5 +55,4 @@ def __init__(self, media): self.media = media self.source = self.media - self.metadata = {} self.file_name, self.file_title, self.file_ext = get_file_info(media) From b778ce16e11561784546e0ee9d71efc07d936fec Mon Sep 17 00:00:00 2001 From: achbed Date: Wed, 28 Jan 2015 00:13:21 -0600 Subject: [PATCH 68/82] Fixes multiple regression issues with new MediaBase class Additional logging on failure to track issues with media classes Signed-off-by: achbed --- deefuzzer/station.py | 30 +++++++++++++++++++++--------- deefuzzer/tools/mediabase.py | 35 ++++++++++++++++++++++------------- deefuzzer/tools/mp3.py | 4 ++-- deefuzzer/tools/ogg.py | 2 +- 4 files changed, 46 insertions(+), 25 deletions(-) diff --git a/deefuzzer/station.py b/deefuzzer/station.py index 6272434..d914779 100644 --- a/deefuzzer/station.py +++ b/deefuzzer/station.py @@ -547,10 +547,10 @@ def media_to_objs(self, media_list): file_meta = Ogg(media) elif file_ext.lower() == 'webm' or mimetypes.guess_type(media)[0] == 'video/webm': file_meta = WebM(media) - except: + except Exception, e: + self._err('Could not get specific media type class for %s' % (media)) + self._err('Error: %s' % (str(e))) pass - file_meta.metadata['filename'] = file_name.decode("utf-8") # decode needed for some weird filenames - file_meta.metadata['filepath'] = media.decode("utf-8") # decode needed for some weird filenames self.q.task_done() media_objs.append(file_meta) return media_objs @@ -589,6 +589,12 @@ def update_feeds(self, media_list, rss_file, sub_title): media_description += media_description_item % (key.capitalize(), media.metadata[key]) json_item[key] = media.metadata[key] + if self.feeds_showfilepath: + media_description += media_description_item % ('Filepath', media.media) + json_item['filepath'] = media.media + if self.feeds_showfilename: + media_description += media_description_item % ('Filename', media.file_name) + json_item['filename'] = media.file_name media_description += '' title, artist, song = self.get_songmeta(media) @@ -686,18 +692,24 @@ def get_currentsongmeta(self): m = self.media_to_objs([self.media]) self.current_media_obj = m[0] except: + self._info("Failed to get media object for %s" % (self.media)) pass self.title, self.artist, self.song = self.get_songmeta(self.current_media_obj) def set_read_mode(self): self.prefix = '#nowplaying' - self.get_currentsongmeta() - - self.metadata_file = self.metadata_dir + os.sep + self.current_media_obj.file_name + '.xml' - self.update_feeds([self.current_media_obj], self.feeds_current_file, '(currently playing)') - self._info('DeeFuzzing: id = %s, name = %s' - % (self.id, self.current_media_obj.file_name)) + + try: + self.get_currentsongmeta() + fn = self.current_media_obj.file_name + if fn: + self.metadata_file = self.metadata_dir + os.sep + fn + '.xml' + self.update_feeds([self.current_media_obj], self.feeds_current_file, '(currently playing)') + if fn: + self._info('DeeFuzzing: id = %s, name = %s' % (self.id, fn)) + except: + pass self.player.set_media(self.media) self.q.get(1) diff --git a/deefuzzer/tools/mediabase.py b/deefuzzer/tools/mediabase.py index ef71132..ee74697 100644 --- a/deefuzzer/tools/mediabase.py +++ b/deefuzzer/tools/mediabase.py @@ -28,12 +28,12 @@ def __init__(self): # tagdata contains a dictionary of tags to use to gather metadata from the sourceobj self.tagdata = {} - self.media = None + self.media = '' self.item_id = '' - self.source = None + self.source = '' self.options = {} self.bitrate_default = 0 - self.info = None + self.info = {} self.bitrate = 0 self.length = 0 @@ -74,10 +74,10 @@ def get_file_metadata(self, clear_cache=False): """Returns the metadata for the media, filtered by the tagdata dictionary for this media type. Return value is read from cache if possible (or unless clear_cache is set to True)""" if not self.metadata or clear_cache: - self.__read_file_metadata() + self.read_file_metadata() return self.metadata - def __read_file_metadata(self): + def read_file_metadata(self): """Reads the metadata for the media, filtered by the tagdata dictionary for this media type""" self.metadata = {} for key in self.tagdata.keys(): @@ -85,18 +85,26 @@ def __read_file_metadata(self): try: self.metadata[key] = self.sourceobj[key][0] except: - try: - if self.tagdata[key] != '': - self.metadata[key] = self.sourceobj[self.tagdata[key]][0] - except: - pass + pass + + try: + if self.tagdata[key] != '' and self.metadata[key] == "": + self.metadata[key] = self.sourceobj[self.tagdata[key]][0] + except: + pass - def get_metadata_value(self, key, clean=False): + def get_metadata_value(self, key, clean=False, clear_cache=False): """Returns a metadata value for a give key. If clean is True, then the resulting string will - be cleaned before it is returned. If the key does not exist, an empty string is returned.""" + be cleaned before it is returned. If the key does not exist, an empty string is returned. Return + value is read from cache if possible (or unless clear_cache is set to True)""" + if not self.metadata or clear_cache: + self.read_file_metadata() + if key not in self.metadata: return '' r = self.metadata[key] + if not r: + r = ""; if clean: r = r.replace('_',' ').strip() return r.encode('utf-8') @@ -116,7 +124,8 @@ def get_song(self, usefn=True): a = self.get_metadata_value('artist', True) t = self.get_metadata_value('title', True) if len(a) == 0 and len(t) == 0 and usefn: - a = self.file_name.encode('utf-8') + if self.file_name: + a = self.file_name.encode('utf-8') r = a if len(a) > 0 and len(t) > 0: r += ' - ' diff --git a/deefuzzer/tools/mp3.py b/deefuzzer/tools/mp3.py index a1cafd0..bc14cd6 100644 --- a/deefuzzer/tools/mp3.py +++ b/deefuzzer/tools/mp3.py @@ -81,14 +81,14 @@ def __init__(self, newmedia): self.bitrate = int(self.info.bitrate / 1024) except: pass - self.length = datetime.timedelta(0, self.info.length) - self.__read_file_metadata() self.media_info = get_file_info(self.media) self.file_name = self.media_info[0] self.file_title = self.media_info[1] self.file_ext = self.media_info[2] self.size = os.path.getsize(self.media) + self.length = datetime.timedelta(0, self.info.length) + self.read_file_metadata() def write_tags(self): """Write all ID3v2.4 tags by mapping dub2id3_dict dictionnary with the diff --git a/deefuzzer/tools/ogg.py b/deefuzzer/tools/ogg.py index 93ec0d8..8f9adea 100644 --- a/deefuzzer/tools/ogg.py +++ b/deefuzzer/tools/ogg.py @@ -72,7 +72,7 @@ def __init__(self, media): self.info = self.sourceobj.info self.bitrate = int(str(self.info.bitrate)[:-3]) self.length = datetime.timedelta(0, self.info.length) - self.__read_file_metadata() + self.read_file_metadata() self.media_info = get_file_info(self.media) self.file_name = self.media_info[0] self.file_title = self.media_info[1] From 399c3f4c256a04ceb18093b2a91e56d4c4b85cdf Mon Sep 17 00:00:00 2001 From: Guillaume Pellerin Date: Wed, 28 Jan 2015 23:54:53 +0100 Subject: [PATCH 69/82] update doc and deps --- README.rst | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index fcccbf4..249cf3a 100644 --- a/README.rst +++ b/README.rst @@ -45,7 +45,7 @@ News 0.7 - * Huge refactoring which should be compatible with old setups, but please read the `updated example `_ before going on + * Huge refactoring which should be compatible with old setups, but before updating **please read** the `updated example `_ and the following news. * Reworked the RSS feed handling to allow JSON output as well and more configuration options (@achbed #27 #28) * Add an init.d script to act as a deamon (@achbed) * Add stationdefaults preference (apply default settings to all stations) (@achbed #31) @@ -86,9 +86,9 @@ an install inside Gygwin should work well. To install it, say on Debian, do:: - sudo apt-get install python-pip python-dev libshout3-dev python-liblo python-mutagen \ - python-pycurl liblo-dev libshout3-dev librtmp-dev \ - python-yaml libcurl4-openssl-dev python-mutagen icecast2 + sudo apt-get install python-pip python-dev python-liblo \ + python-mutagen python-pycurl python-yaml \ + libshout3-dev librtmp-dev liblo-dev libcurl4-openssl-dev Now, the easiest way to install the DeeFuzzer from a shell:: @@ -100,6 +100,10 @@ To upgrade:: For more informations, please see on `GitHub `_ or tweet to @yomguy +As a streaming client, the DeeFuzzer needs a local or remote streaming server like Icecast2 to do something:: + + sudo apt-get install icecast2 + Usage ===== From 1e7c7c3533972e517d08d0b7261a4fb190cf3dd8 Mon Sep 17 00:00:00 2001 From: achbed Date: Thu, 29 Jan 2015 12:57:11 -0600 Subject: [PATCH 70/82] Improper conversion type when reading maxretry parameter Signed-off-by: achbed --- deefuzzer/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deefuzzer/core.py b/deefuzzer/core.py index ad84834..2a6d446 100644 --- a/deefuzzer/core.py +++ b/deefuzzer/core.py @@ -89,7 +89,7 @@ def __init__(self, conf_file): elif key == 'maxretry': # Maximum number of attempts to restart the stations on crash. - self.maxretry = bool(self.conf['deefuzzer'][key]) + self.maxretry = int(self.conf['deefuzzer'][key]) elif key == 'station': # Load station definitions from the main config file From d2ee9b20609f4e9a11d918c267d822c4dc4aebdd Mon Sep 17 00:00:00 2001 From: Guillaume Pellerin Date: Thu, 29 Jan 2015 22:53:36 +0100 Subject: [PATCH 71/82] update features and install procedure --- README.rst | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index fcccbf4..ffab9e9 100644 --- a/README.rst +++ b/README.rst @@ -24,7 +24,7 @@ Features * MP3, OGG Vorbis file and live streaming over Internet * Full metadata encapsulation and management - * RSS podcast generator (current tracks and playlists) + * RSS or JSON podcast generator (current tracks and playlists) * M3U playlist generator * Recursive, random (shuffled) or pre-defined playlists * Multi-threaded architecture: multiple station streaming with one config file @@ -33,6 +33,7 @@ Features * OSC controller: control the main functions from a distant terminal * Relaying: relay and stream live sessions! * WebM video relaying support + * Automatic mountpoint creation based on media subfolders * Very light and optimized streaming process * Fully written in Python * Works with Icecast2, ShoutCast, Stream-m @@ -90,7 +91,11 @@ To install it, say on Debian, do:: python-pycurl liblo-dev libshout3-dev librtmp-dev \ python-yaml libcurl4-openssl-dev python-mutagen icecast2 -Now, the easiest way to install the DeeFuzzer from a shell:: +Now update distribute and setuptools:: + + sudo pip install -U distribute setuptools + +Then:: sudo pip install deefuzzer @@ -98,6 +103,8 @@ To upgrade:: sudo pip install -U deefuzzer +If you have some version problems with the installation, please try in a virtualenv. + For more informations, please see on `GitHub `_ or tweet to @yomguy From 0cc85a3ca232e8870b8b4749744f415e51d57c66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20LASSERRE?= Date: Thu, 29 Jan 2015 23:08:50 +0100 Subject: [PATCH 72/82] Update README.rst Quick fix ;) --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index fcccbf4..bea4e3e 100644 --- a/README.rst +++ b/README.rst @@ -57,8 +57,8 @@ News * Better thread management (@achbed #36 #37 #38) * Improved stability avoiding crashes with automatic station restart methods (@achbed #39 #45) * Added option (ignoreerrors) to log and continue when an error occurs during station setup (@achbed #43) - * Cleanup, better doucmentation and good ideas (@choiz #15 #16 #17 #23) - * Various bufixes + * Cleanup, better documentation and good ideas (@choiz #15 #16 #17 #23) + * Various bugfixes * Many thanks to all participants and especially to @achbed for his **huge** work, efficiency and easy collaboration * Enjoy! From b1f78e4ceddb60f46f3eef93bfec80cccd7642b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20LASSERRE?= Date: Thu, 29 Jan 2015 23:12:15 +0100 Subject: [PATCH 73/82] Update README.rst automaTically instead of automaGically (but it can be magical ;)) --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index fcccbf4..463453f 100644 --- a/README.rst +++ b/README.rst @@ -49,7 +49,7 @@ News * Reworked the RSS feed handling to allow JSON output as well and more configuration options (@achbed #27 #28) * Add an init.d script to act as a deamon (@achbed) * Add stationdefaults preference (apply default settings to all stations) (@achbed #31) - * Add stationfolder preference (generate stations automagically from a folder structure) (@achbed #31) + * Add stationfolder preference (generate stations automatically from a folder structure) (@achbed #31) * Add stationconfig preference (load other preference files as stations) (@achbed #31) * Add new station.server.appendtype option * Add new base_dir parameter to station definition From 7cb3ca99c786ed7c997c1ae4e00cae06b873d7b2 Mon Sep 17 00:00:00 2001 From: achbed Date: Fri, 30 Jan 2015 03:22:02 -0600 Subject: [PATCH 74/82] Changed default for appendtype to 0 (from 1) Signed-off-by: achbed --- deefuzzer/station.py | 4 ++-- example/deefuzzer_doc.xml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/deefuzzer/station.py b/deefuzzer/station.py index d914779..c926f11 100644 --- a/deefuzzer/station.py +++ b/deefuzzer/station.py @@ -1,4 +1,4 @@ -#!/usr/bin/python + #!/usr/bin/python # -*- coding: utf-8 -*- # # Copyright (C) 2006-2011 Guillaume Pellerin @@ -74,7 +74,7 @@ class Station(Thread): relay_mode = 0 record_mode = 0 run_mode = 1 - appendtype = 1 + appendtype = 0 feeds_json = 0 feeds_rss = 1 feeds_mode = 1 diff --git a/example/deefuzzer_doc.xml b/example/deefuzzer_doc.xml index 20b68d0..93ed50d 100644 --- a/example/deefuzzer_doc.xml +++ b/example/deefuzzer_doc.xml @@ -160,8 +160,8 @@ icecast - 1 + '0' will leave the mount name alone. Default is 0. Used only for icecast streams --> + 0 From 47554bd6950d25a9980a177d472b6acfae453028 Mon Sep 17 00:00:00 2001 From: achbed Date: Fri, 30 Jan 2015 03:32:43 -0600 Subject: [PATCH 75/82] Removed redundant reference to stationdefaults Changed from "station setup" to "station initialization" for better clarity Signed-off-by: achbed --- README.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.rst b/README.rst index b13e47f..b23be9a 100644 --- a/README.rst +++ b/README.rst @@ -54,10 +54,9 @@ News * Add stationconfig preference (load other preference files as stations) (@achbed #31) * Add new station.server.appendtype option * Add new base_dir parameter to station definition - * New stationdefaults parameter (provides a mechanism to give default values for all stations) (@achbed #29) * Better thread management (@achbed #36 #37 #38) * Improved stability avoiding crashes with automatic station restart methods (@achbed #39 #45) - * Added option (ignoreerrors) to log and continue when an error occurs during station setup (@achbed #43) + * Added option (ignoreerrors) to log and continue when an error occurs during station initialization (@achbed #43) * Cleanup, better documentation and good ideas (@choiz #15 #16 #17 #23) * Various bugfixes * Many thanks to all participants and especially to @achbed for his **huge** work, efficiency and easy collaboration From 2692dbd03da1f5326d814de4facf45cba638e9fd Mon Sep 17 00:00:00 2001 From: achbed Date: Fri, 30 Jan 2015 03:37:57 -0600 Subject: [PATCH 76/82] Fix for extra indent on line 1 of station.py --- deefuzzer/station.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deefuzzer/station.py b/deefuzzer/station.py index c926f11..748abb8 100644 --- a/deefuzzer/station.py +++ b/deefuzzer/station.py @@ -1,4 +1,4 @@ - #!/usr/bin/python +#!/usr/bin/python # -*- coding: utf-8 -*- # # Copyright (C) 2006-2011 Guillaume Pellerin From 1794b67582fd8da1b8f67463db08ceedac9fed3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20LASSERRE?= Date: Fri, 30 Jan 2015 22:17:24 +0100 Subject: [PATCH 77/82] Update station.py Fix m3u listing files. --- deefuzzer/station.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deefuzzer/station.py b/deefuzzer/station.py index 748abb8..af4cb02 100644 --- a/deefuzzer/station.py +++ b/deefuzzer/station.py @@ -291,7 +291,7 @@ def _path_add_base(self, a): return os.path.join(self.base_directory, a) def _path_m3u_rel(self, a): - return os.path.join(os.path.dirname(self.source), a) + return os.path.join(os.path.dirname(self.media_source), a) def _log(self, level, msg): try: From ac3432680095e6f7f2648b751413e5b7fc4ad18d Mon Sep 17 00:00:00 2001 From: Guillaume Pellerin Date: Sat, 31 Jan 2015 00:23:22 +0100 Subject: [PATCH 78/82] Put optional doc in the FAQ --- README.rst | 133 ++++++----------------------------------------------- 1 file changed, 14 insertions(+), 119 deletions(-) diff --git a/README.rst b/README.rst index 3e3dd0a..d67e718 100644 --- a/README.rst +++ b/README.rst @@ -22,18 +22,16 @@ live multimedia relays or personal home radios, with metadata management and coo Features ======== - * MP3, OGG Vorbis file and live streaming over Internet + * Streaming MP3, OGG Vorbis files over Internet + * Live streaming for any kind of format (WebM compatible) * Full metadata encapsulation and management - * RSS or JSON podcast generator (current tracks and playlists) - * M3U playlist generator - * Recursive, random (shuffled) or pre-defined playlists - * Multi-threaded architecture: multiple station streaming with one config file - * Auto twitting the current playing track and new tracks + * Recursive folders, random or M3U playlists management + * M3U, RSS and JSON podcast generators for URLs, current tracks and playlists + * Automagic mountpoint creation based on media subfolders + * Multiple station streaming with only one config file + * Auto twitting #nowplaying tracks * Auto jingling between tracks - * OSC controller: control the main functions from a distant terminal - * Relaying: relay and stream live sessions! - * WebM video relaying support - * Automatic mountpoint creation based on media subfolders + * OSC controller for a fex commands * Very light and optimized streaming process * Fully written in Python * Works with Icecast2, ShoutCast, Stream-m @@ -46,7 +44,7 @@ News 0.7 - * Huge refactoring which should be compatible with old setups, but before updating **please read** the `updated example `_ and the following news. + * **Huge** refactoring which should be compatible with old setups, but before updating **please read** the `updated example `_ and the following news. * Reworked the RSS feed handling to allow JSON output as well and more configuration options (@achbed #27 #28) * Add an init.d script to act as a deamon (@achbed) * Add stationdefaults preference (apply default settings to all stations) (@achbed #31) @@ -59,7 +57,7 @@ News * Improved stability avoiding crashes with automatic station restart methods (@achbed #39 #45) * Added option (ignoreerrors) to log and continue when an error occurs during station setup (@achbed #43) * Cleanup, better doucmentation and good ideas (@choiz #15 #16 #17 #23) - * Various bufixes + * Many bufixes * Many thanks to all participants and especially to @achbed for his **huge** work, efficiency and easy collaboration * Enjoy! @@ -143,7 +141,7 @@ or, more specificially:: pkill -9 -f "deefuzzer example/deefuzzer.yaml" -Configuration +doucmentation ============= Some examples of markup configuration files: @@ -154,107 +152,6 @@ Some examples of markup configuration files: * `generic YAML `_ -OSC Control -=========== - -Some of the DeeFuzzer function parameters can be control through the great OSC protocol. -The OSC server is only active if the tag is set up to "1" -in the config file (see example/deefuzzer.xml again..). - -The available parameters are: - - * playing: next track - * twitting: start and stop - * relaying: start and stop - * jingling: start and stop - * recording: start and stop - -See `examples here. `_ - -Then any OSC remote (PureDate, Monome, TouchOSC, etc..) can a become controller! :) - -We provide some client python scripts as some examples about how to control the parameters -from a console or any application (see deefuzzer/scripts/). - - -Twitter (manual and optional) -============================= - -To get track twitting, please install python-twitter, python-oauth2 and all their dependencies. - -Install or make sure python-oauth2 and python-twitter are installed:: - - sudo easy_install oauth2 - sudo pip install python-twitter - -As Twitter access requires oauth keys since 07/2010, you need to get your own access token key pair. -Please run the dedicated script to do this from the main deefuzzer app directory:: - - python tools/get_access_token.py - -You will be invited to copy/paste an URL in your browser to get a pin code. -Then copy/paste this code into the console and press ENTER. -The script gives you a pair of keys: one access token key and one access token secret key. - -Change the block options in your deefuzzer XML config file, giving the 2 keys. -For example:: - - - 1 - 85039615-H6yAtXXCx7NobF5W40FV0c8epGZsQGkE7MG6XRjD2 - A1YW3llB9H9qVbjH8zOQTOkMlhVqh2a7LnA9Lt0b6Gc - Music Groove - - -Your DeeFuzzer will now tweet the currently playing track and new tracks on your profile. - - -Station Folders -=============== - -Station folders are a specific way of setting up your file system so that you can auto-create many stations -based on only a few settings. The feature requires a single main folder, with one or more subfolders. Each -subfolder is scanned for the presence of media files (audio-only at the moment). If files are found, then a -station is created using the parameters in the block. Substitution is performed to fill in -some detail to the stationfolder parameters, and all stationdefaults are also applied. - -The base folder is specified by the block. No substitution is done on this parameter. - -Subsitution is done for [name] and [path] - [name] is replaced with the name of the subfolder, and [path] is -replaced with the subfolder's complete path. - -Consider the following example. We have a block with the following settings: - - - /path/to/media - - [name] - [name] - [name] - - - [path] - - - -The folder structure is as follows: - - /path/to/media - + one - - song1.mp3 - - song2.mp3 - + two - - song3.ogg - + three - - presentation.pdf - + four - - song4.mp3 - -In this case, three stations are created: one, two, and four. Each will have their short name (and thus their -icecast mount point) set to their respective folder names. Subfolder three is skipped, as there are no audio files -present - just a PDF file. - - API === @@ -265,9 +162,10 @@ Development =========== Everybody is welcome to participate to the DeeFuzzer project! -We use GitHub to collaborate: https://github.com/yomguy/DeeFuzzer -Clone it, star it, join us! +We use GitHub to collaborate: https://github.com/yomguy/DeeFuzzer + +Clone it, star it and join us! Authors @@ -301,7 +199,4 @@ Contact / Infos =============== Twitter: @yomguy @parisson_studio - -GitHub: https://github.com/yomguy/DeeFuzzer - Expertise, Business, Sponsoring: http://parisson.com From 6b5a33c9f3ca187f5379d58a40611457cd658a93 Mon Sep 17 00:00:00 2001 From: Guillaume Pellerin Date: Sat, 31 Jan 2015 00:29:29 +0100 Subject: [PATCH 79/82] cleanup README (pushing some part in the wiki) --- README.rst | 33 +++++++++------------------------ 1 file changed, 9 insertions(+), 24 deletions(-) diff --git a/README.rst b/README.rst index 1dc5f91..8a19bab 100644 --- a/README.rst +++ b/README.rst @@ -127,34 +127,19 @@ To make the deefuzzer act as a deamon, just play it in the background:: deefuzzer example/deefuzzer.yaml & Note that you must edit the config file with right parameters before playing. -You can find an example for a draft XML file in the "example" directory of the source code. -WARNING: because we need the DeeFuzzer to be a very stable streaming process with multiple channel management, -the multi-threaded implementation of deefuzzer instances avoids shutting down the process with a CTRL+C. -You have to kill them manually, after a CTRL+Z, making this:: - pkill -9 deefuzzer - -or, more specificially:: - - pkill -9 -f "deefuzzer example/deefuzzer.yaml" - - -doucmentation +Documentation ============= -Some examples of markup configuration files: - - * `generic and documented XML `_ - * `generic XML for testing `_ - * `OGG Vorbis and MP3 together `_ - * `generic YAML `_ - - -API -=== - -http://files.parisson.com/doc/deefuzzer/ + * `Wiki `_ + * `API `_ + * `Documented XML configuration `_ + * Configuration examples: + + * `Dummy XML for testing `_ + * `OGG Vorbis and MP3 together `_ + * `Generic YAML `_ Development From f39ba066fe5500e7adab488c4723c953a5c1b30d Mon Sep 17 00:00:00 2001 From: Guillaume Pellerin Date: Sat, 31 Jan 2015 00:36:50 +0100 Subject: [PATCH 80/82] cleanup --- README.rst | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 8a19bab..c6428dd 100644 --- a/README.rst +++ b/README.rst @@ -100,9 +100,7 @@ To upgrade:: sudo pip install -U deefuzzer -If you have some version problems with the installation, please try in a virtualenv. - -For more informations, please see on `GitHub `_ or tweet to @yomguy +If you have some version problems with the installation, please also try in a virtualenv. As a streaming client, the DeeFuzzer needs a local or remote streaming server like Icecast2 to do something:: From 953f9e9d0ae469bb187214c703e3b0b32565a6d2 Mon Sep 17 00:00:00 2001 From: Guillaume Pellerin Date: Sat, 31 Jan 2015 00:37:35 +0100 Subject: [PATCH 81/82] fix doc --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index c6428dd..00a1ff7 100644 --- a/README.rst +++ b/README.rst @@ -130,7 +130,7 @@ Note that you must edit the config file with right parameters before playing. Documentation ============= - * `Wiki `_ + * `FAQ and Wiki `_ * `API `_ * `Documented XML configuration `_ * Configuration examples: From 170a51e98d394e981dc951aa1deeeb38bf8192b9 Mon Sep 17 00:00:00 2001 From: Guillaume Pellerin Date: Sat, 31 Jan 2015 00:44:06 +0100 Subject: [PATCH 82/82] update MANIFEST --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) diff --git a/MANIFEST.in b/MANIFEST.in index 2c6e3a9..f5cdcfe 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,6 @@ include *.txt include *.rst +include *.md exclude *.cfg recursive-include doc * recursive-include example *