keywords2.py

#! /opt/local/bin/python2.7 -tt
# -*- coding: utf-8 -*-

import os
import re
import sys
import unicodedata
import json
import gzip
from datetime import datetime, timedelta 
import time

kw1 = [ 'directeur', 'recherche', 'Inria', 'équipe', 'Montpellier']

kw2 = [ 'enjeu', 'économique', '(cent|100)', '(milliard|md)(s)?', '(dollars|$)', 
        'infini', 'calcul', 'stockage', 'réseau', '(data( )?center|centre( )?(de )? donnée)',
        'dématériali', '(The)? Network (is )?(the )?Computer', 'réseau (est )?(l )?ordinateur',  
        'serveurs', 'puissance' ]

kw3 = [ 'particulier', 'service', 'courrier(s)? électronique', 'agenda', 'document', 'travail collaboratif', 
        'entreprise', 'centre(s)? (de )?donnée', 'petite(s)?( et)? moyenne', 'PME', 'moyen',
        'infrastructure', 'IaaS', 'Infrastructure as a Service', 'ASP', 'Application Service Provider', 'fournisseur',
        'Platform as a Service', 'PaaS', 'plat(e)?forme (de )?calcul', 'API', 'Software as a Service', 'Google' ]

kw4 = [ '(réduction|réduire)', 'coût', 'PME', 'infrastructure', 'fournisseur', 'payer', 'consommer', 'mutualis', 
        'gestion', 'possession', 'opération', 'faciliter',
        'mobile', 'qualité', 'service', 'élasticité', 'pic', ' charge', 'application', 'paiement',  
        'surcharge', 'puissance', 'calcul', 'stockage' ] 
 
kw5 = [ 'grande(s)? entreprise', 'système', 'information', 'transactionnel', 'bancaire', 'réservation', 'décisionnelle', 'cohérence', 
        'temps', 'réponse', 'protection', 'opérateur', 'confiance', 'enjeu', 'interopérabilité', 'Microsoft', 
        'IBM', 'Google', 'Amazon', 'réseau(x)? sociau(x)?', 'compatib(le|ilité)', 'otage', 'fournisseur', 'standard', 'récupérer', 
        'confidentialité', 'Facebook', 'volume', 'donnée', 'échelle', 'architecture', 'Autonomic Computing', 'Green Computing', 
        'informatique verte', 'consommer', 'énergie', 'mutualis' ]

kw6 = [ 'programm', 'application', 'industrie', 'communauté', 'distribué', 'exploitation', 
        'gestion (de )?donnée', 'réseau(x)? sociau', 'sociologue', 'Boom', 'langage(s)? (de )?(programmation)? déclarati', '(base(s)? (de )?donnée|bd)' ]

delays = {
	'001': 0,
	'002': 4,
	'003': 8,
	'004': 4,
	'005': 8,
	'006': 0,
	'007': 6,
	'008': 6,
	'009': 10,
	'010': 10,
	'012': 6,
	'013': 10,
	'014': 4,
	'016': 8,
	'017': 0,
	'018': 4,
	'019': 0,
	'020': 8,
	'021': 6,
	'025': 10
}


def load_text_file(filename):
	with open(filename, "r") as text_file:
	    text = text_file.read()
	return text

def load_pad_revisions(db_file, padid):
	with gzip.open(db_file, 'rb') as f:		
		revisions = []
		pattern_rev_metadata = re.compile('{"key":"pad:'+padid+':revs:([0-9]+)"')
		pattern_rev_data = re.compile('pad:'+padid)

		while True:
			try:
				line = f.next()
				match = pattern_rev_metadata.match(line)
				if match:
					rev = int(match.group(1)) #-> to int ?
					rev_partA = json.loads(line)
					rev_partB = json.loads(f.next())
					if not pattern_rev_data.match(rev_partB['key']):
						raise Exception('missing pad content for rev:%i' % (rev))
					if rev_partB['val']['head'] != rev:
						raise Exception( 'head (%i) does not match with current rev (%i)' % (rev_partB['val']['head'], rev))
					revision = {}			
					revision['rev'] = rev
					revision['timestamp'] = rev_partA['val']['meta']['timestamp']
					revision['datetime'] = datetime.fromtimestamp(int(revision['timestamp']) / 1000)
					revision['author'] = rev_partA['val']['meta']['author']
					revision['content'] = rev_partB['val']['atext']['text']

					revisions.append(revision)

			except StopIteration:
				break
		return revisions

def format_time(timestamp):
	return timestamp.strftime('%H:%M:%S')

def get_revision(revisions, revision_num):
	return [ rev for rev in revisions if rev['rev'] == revision_num ][0]

def get_revision_at_time(revisions, certain_time):
	return [ rev for rev in revisions if rev['datetime'] <= certain_time ][-1]


SPLIT_MARKERS = [ '1. Cloud computing - concept innovateur \(Utilisateur 1 \+ Utilisateur 2\)',
                  '2. Différents types de clouds et de clients \(Utilisateur 3 \+ Utilisateur 4\)',
                  '3. Les avantages de cloud \(Utilisateur 1 \+ Utilisateur 2\)',
                  '4. Les inconvénients de cloud \(Utilisateur 3 \+ Utilisateur 4\)',
                  '5. Sujets de recherche en cloud computing \(Utilisateur 1 \+ Utilisateur 2\)' ]

SPLIT_MARKERS = [ '1. Cloud computing - concept innovateur \(Utilisateur 1 \+ Utilisateur 2\)',
                  '2. Diff.rents types de clouds et de clients \(Utilisateur 3 \+ Utilisateur 4\)',
                  '3. Les avantages de cloud \(Utilisateur 1 \+ Utilisateur 2\)',
                  '4. Les inconv.nients de cloud \(Utilisateur 3 \+ Utilisateur 4\)',
                  '5. Sujets de recherche en cloud computing \(Utilisateur 1 \+ Utilisateur 2\)' ]

UNESCAPED_SPLIT_MARKERS = [ m.replace('\\','') for m in SPLIT_MARKERS ]

def remove_markers(some_text):
	for marker in UNESCAPED_SPLIT_MARKERS:
		#some_text = some_text.replace(marker, '')
		some_text = re.sub(marker, '', some_text)
		
	return some_text

def remove_accents(input_str):
	nkfd_form = unicodedata.normalize('NFKD', input_str)
	return u"".join([c for c in nkfd_form if not unicodedata.combining(c)])

def uniformize_and_clean(some_text):
	some_text = some_text.decode(TEXT_ENCODING)
	some_text = remove_markers(some_text)
	some_text = some_text.lower()

	# Remove accents (replaced with the corresponding non-accented character)
	some_text = remove_accents(some_text)

	# Separate most punctuation
	some_text = re.sub(r"([^\w\.\'\-\/,&])", r' \1 ', some_text)

	# Separate commas if they're followed by space.
	# (E.g., don't separate 2,500)
	some_text = re.sub(r"(,\s)", r' \1', some_text)

	# Separate single quotes if they're followed by a space.
	# some_text = re.sub(r"('\s)", r' \1', some_text)

	# Separate single quotes
	some_text = re.sub(r"(\w)'(\w)", r"\1 ' \2", some_text)

	# Separate 'x/x' into 'x / x'
	some_text = re.sub(r"(\w)/(\w)", r'\1 / \2', some_text)

	# Separate periods that come before newline or end of string.
	some_text = re.sub('\. *(\n|$)', ' . ', some_text)

	# Remove 'punctuations' signs
	some_text = re.sub('[\.\'\-\/,&>\*=\+\(\):;\"\'\[\]\?!$~\^\|]', '', some_text)
	
	return some_text


def get_uniformized_keywords(): 
	some_keywords = kw1 + kw2 + kw3 + kw4 + kw5 + kw6
	some_keywords = [remove_accents(kw.decode(TEXT_ENCODING)).lower() for kw in some_keywords]
	some_keywords = list(set(some_keywords))
	return some_keywords 



INPUT_DATA_PATH='./DATA-by-num/'
INPUT_DATA_JSON_FILE='./chat-slicing-data-notes.json'
TEXT_ENCODING="utf-8"

with open(INPUT_DATA_JSON_FILE, "r") as json_data_file:
	data = json.loads(json_data_file.read())

	print '; group, delay (in s.), matching_keywords, non_matching_keywords '
	for group in sorted(data.keys()):
	#for group in ['019']:
		for experiment in data[group].keys():
			if group == '014' and (experiment == "corrections" or experiment == "films"):
				num = '015'
			else:
				num = group
  
		revisions = load_pad_revisions(INPUT_DATA_PATH + num + '/dirty.db.gz', experiment + num)

		initial_doc_rev_num = data[group][experiment]["init-rev"]
		first_changes_rev_num = data[group][experiment]["first-change-rev"]
		end_of_audio_rev_num = data[group][experiment]["end-of-audio-rev"]

		initial_doc_rev = get_revision(revisions, initial_doc_rev_num)
		first_changes_rev = get_revision(revisions, first_changes_rev_num)
		end_of_audio_rev = get_revision(revisions, end_of_audio_rev_num)

		# We consider only one revision at 5 minutes after end-of-audio
		considered_revision = get_revision_at_time(revisions, end_of_audio_rev['datetime'] + timedelta(minutes=5))
		
		text = considered_revision['content'].encode(TEXT_ENCODING)
		text = uniformize_and_clean(text)
		
		# Check if there still exist some non-words characters
		non_word_characters = set(re.sub('[A-Za-z0-9\s]', '', text))
		if (len(non_word_characters) > 0):
			print 'WARNING: seems that there are still some unexpected non-in-a-word characters - ', non_word_characters

		text = re.sub('\s+', ' ', text)

		ok = 0
		pas_ok = 0

		keywords = get_uniformized_keywords()
		for kw in keywords:
			pattern = re.compile(kw)

			if (pattern.search(text)):
				ok = ok + 1
			else:
				pas_ok = pas_ok + 1
				#print '***', kw

		print ",".join(map(lambda x: str(x), [int(num), delays[num], ok, pas_ok]))